2024-11-14 02:43:05,984 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-14 02:43:05,995 main DEBUG Took 0.009018 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-14 02:43:05,995 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-14 02:43:05,996 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-14 02:43:05,997 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-14 02:43:05,998 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:43:06,005 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-14 02:43:06,016 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:06,017 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:43:06,018 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:06,019 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:43:06,019 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:06,019 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:43:06,020 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:06,021 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:43:06,021 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:06,021 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:43:06,022 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:06,022 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:43:06,023 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:06,023 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:43:06,024 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:06,024 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:43:06,025 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:06,025 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:43:06,025 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:06,026 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:43:06,026 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:06,026 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:43:06,027 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:06,027 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:43:06,028 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:06,028 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-14 02:43:06,029 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:06,030 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-14 02:43:06,032 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-14 02:43:06,033 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-14 02:43:06,034 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-14 02:43:06,034 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-14 02:43:06,041 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-14 02:43:06,043 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-14 02:43:06,045 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-14 02:43:06,045 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-14 02:43:06,046 main DEBUG createAppenders(={Console}) 2024-11-14 02:43:06,047 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-14 02:43:06,047 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-14 02:43:06,047 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-14 02:43:06,048 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-14 02:43:06,048 main DEBUG OutputStream closed 2024-11-14 02:43:06,048 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-14 02:43:06,048 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-14 02:43:06,049 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-14 02:43:06,117 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-14 02:43:06,119 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-14 02:43:06,120 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-14 02:43:06,121 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-14 02:43:06,122 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-14 02:43:06,122 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-14 02:43:06,122 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-14 02:43:06,123 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-14 02:43:06,123 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-14 02:43:06,123 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-14 02:43:06,123 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-14 02:43:06,124 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-14 02:43:06,124 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-14 02:43:06,124 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-14 02:43:06,125 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-14 02:43:06,125 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-14 02:43:06,125 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-14 02:43:06,126 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-14 02:43:06,128 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14 02:43:06,129 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-14 02:43:06,129 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-14 02:43:06,130 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-14T02:43:06,339 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb 2024-11-14 02:43:06,342 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-14 02:43:06,342 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14T02:43:06,350 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-14T02:43:06,369 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-14T02:43:06,384 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/cluster_801a70f6-fc7b-44f6-e8da-0c348f54bc49, deleteOnExit=true 2024-11-14T02:43:06,385 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-14T02:43:06,386 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/test.cache.data in system properties and HBase conf 2024-11-14T02:43:06,386 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/hadoop.tmp.dir in system properties and HBase conf 2024-11-14T02:43:06,387 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/hadoop.log.dir in system properties and HBase conf 2024-11-14T02:43:06,388 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-14T02:43:06,388 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-14T02:43:06,389 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-14T02:43:06,470 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-14T02:43:06,560 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T02:43:06,564 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-14T02:43:06,565 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-14T02:43:06,565 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-14T02:43:06,566 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T02:43:06,566 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-14T02:43:06,566 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-14T02:43:06,567 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T02:43:06,567 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T02:43:06,568 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-14T02:43:06,568 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/nfs.dump.dir in system properties and HBase conf 2024-11-14T02:43:06,568 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/java.io.tmpdir in system properties and HBase conf 2024-11-14T02:43:06,569 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T02:43:06,569 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-14T02:43:06,570 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-14T02:43:07,578 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-14T02:43:07,653 INFO [Time-limited test {}] log.Log(170): Logging initialized @2309ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-14T02:43:07,723 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T02:43:07,788 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T02:43:07,810 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T02:43:07,811 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T02:43:07,812 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T02:43:07,826 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T02:43:07,829 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/hadoop.log.dir/,AVAILABLE} 2024-11-14T02:43:07,830 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T02:43:08,012 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/java.io.tmpdir/jetty-localhost-35697-hadoop-hdfs-3_4_1-tests_jar-_-any-721873294562600747/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T02:43:08,018 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:35697} 2024-11-14T02:43:08,018 INFO [Time-limited test {}] server.Server(415): Started @2675ms 2024-11-14T02:43:08,514 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T02:43:08,524 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T02:43:08,525 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T02:43:08,525 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T02:43:08,525 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T02:43:08,526 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/hadoop.log.dir/,AVAILABLE} 2024-11-14T02:43:08,527 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T02:43:08,640 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/java.io.tmpdir/jetty-localhost-40187-hadoop-hdfs-3_4_1-tests_jar-_-any-17159292926082696254/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T02:43:08,641 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:40187} 2024-11-14T02:43:08,641 INFO [Time-limited test {}] server.Server(415): Started @3297ms 2024-11-14T02:43:08,693 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T02:43:09,612 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/cluster_801a70f6-fc7b-44f6-e8da-0c348f54bc49/data/data2/current/BP-2060825341-172.17.0.2-1731552187113/current, will proceed with Du for space computation calculation, 2024-11-14T02:43:09,612 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/cluster_801a70f6-fc7b-44f6-e8da-0c348f54bc49/data/data1/current/BP-2060825341-172.17.0.2-1731552187113/current, will proceed with Du for space computation calculation, 2024-11-14T02:43:09,637 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T02:43:09,678 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbfcc64d3d4e0f4c8 with lease ID 0xa2b8f00b43e8b36: Processing first storage report for DS-f7062220-044e-4c97-9b84-7d0dfdb6f9dc from datanode DatanodeRegistration(127.0.0.1:34599, datanodeUuid=bc2a2627-d085-47a0-9e50-6effdfcfabbd, infoPort=33069, infoSecurePort=0, ipcPort=42495, storageInfo=lv=-57;cid=testClusterID;nsid=1168526170;c=1731552187113) 2024-11-14T02:43:09,680 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbfcc64d3d4e0f4c8 with lease ID 0xa2b8f00b43e8b36: from storage DS-f7062220-044e-4c97-9b84-7d0dfdb6f9dc node DatanodeRegistration(127.0.0.1:34599, datanodeUuid=bc2a2627-d085-47a0-9e50-6effdfcfabbd, infoPort=33069, infoSecurePort=0, ipcPort=42495, storageInfo=lv=-57;cid=testClusterID;nsid=1168526170;c=1731552187113), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-14T02:43:09,680 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbfcc64d3d4e0f4c8 with lease ID 0xa2b8f00b43e8b36: Processing first storage report for DS-37654846-9913-49bf-ae74-68d41cc23b82 from datanode DatanodeRegistration(127.0.0.1:34599, datanodeUuid=bc2a2627-d085-47a0-9e50-6effdfcfabbd, infoPort=33069, infoSecurePort=0, ipcPort=42495, storageInfo=lv=-57;cid=testClusterID;nsid=1168526170;c=1731552187113) 2024-11-14T02:43:09,680 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbfcc64d3d4e0f4c8 with lease ID 0xa2b8f00b43e8b36: from storage DS-37654846-9913-49bf-ae74-68d41cc23b82 node DatanodeRegistration(127.0.0.1:34599, datanodeUuid=bc2a2627-d085-47a0-9e50-6effdfcfabbd, infoPort=33069, infoSecurePort=0, ipcPort=42495, storageInfo=lv=-57;cid=testClusterID;nsid=1168526170;c=1731552187113), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T02:43:09,752 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb 2024-11-14T02:43:09,814 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/cluster_801a70f6-fc7b-44f6-e8da-0c348f54bc49/zookeeper_0, clientPort=60414, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/cluster_801a70f6-fc7b-44f6-e8da-0c348f54bc49/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/cluster_801a70f6-fc7b-44f6-e8da-0c348f54bc49/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T02:43:09,822 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60414 2024-11-14T02:43:09,832 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T02:43:09,834 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T02:43:10,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741825_1001 (size=7) 2024-11-14T02:43:10,433 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b with version=8 2024-11-14T02:43:10,433 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/hbase-staging 2024-11-14T02:43:10,514 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-14T02:43:10,734 INFO [Time-limited test {}] client.ConnectionUtils(128): master/83c5a5c119d5:0 server-side Connection retries=45 2024-11-14T02:43:10,743 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T02:43:10,744 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T02:43:10,748 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T02:43:10,749 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T02:43:10,749 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T02:43:10,869 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T02:43:10,918 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-14T02:43:10,926 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-14T02:43:10,930 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T02:43:10,951 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 116844 (auto-detected) 2024-11-14T02:43:10,952 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-14T02:43:10,969 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43239 2024-11-14T02:43:10,989 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43239 connecting to ZooKeeper ensemble=127.0.0.1:60414 2024-11-14T02:43:11,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:432390x0, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T02:43:11,072 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43239-0x10138834f4e0000 connected 2024-11-14T02:43:11,146 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T02:43:11,149 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T02:43:11,162 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T02:43:11,167 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b, hbase.cluster.distributed=false 2024-11-14T02:43:11,190 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T02:43:11,194 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43239 2024-11-14T02:43:11,195 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43239 2024-11-14T02:43:11,195 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43239 2024-11-14T02:43:11,196 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43239 2024-11-14T02:43:11,197 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43239 2024-11-14T02:43:11,292 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/83c5a5c119d5:0 server-side Connection retries=45 2024-11-14T02:43:11,293 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T02:43:11,293 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T02:43:11,293 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T02:43:11,293 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T02:43:11,294 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T02:43:11,296 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T02:43:11,298 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T02:43:11,299 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41455 2024-11-14T02:43:11,301 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41455 connecting to ZooKeeper ensemble=127.0.0.1:60414 2024-11-14T02:43:11,302 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T02:43:11,305 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T02:43:11,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:414550x0, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T02:43:11,320 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41455-0x10138834f4e0001 connected 2024-11-14T02:43:11,320 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:414550x0, quorum=127.0.0.1:60414, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T02:43:11,325 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T02:43:11,336 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T02:43:11,339 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41455-0x10138834f4e0001, quorum=127.0.0.1:60414, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T02:43:11,345 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41455-0x10138834f4e0001, quorum=127.0.0.1:60414, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T02:43:11,346 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41455 2024-11-14T02:43:11,347 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41455 2024-11-14T02:43:11,349 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41455 2024-11-14T02:43:11,350 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41455 2024-11-14T02:43:11,350 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41455 2024-11-14T02:43:11,365 DEBUG [M:0;83c5a5c119d5:43239 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;83c5a5c119d5:43239 2024-11-14T02:43:11,366 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/83c5a5c119d5,43239,1731552190584 2024-11-14T02:43:11,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41455-0x10138834f4e0001, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T02:43:11,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T02:43:11,379 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/83c5a5c119d5,43239,1731552190584 2024-11-14T02:43:11,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41455-0x10138834f4e0001, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T02:43:11,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:43:11,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41455-0x10138834f4e0001, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:43:11,404 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T02:43:11,405 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/83c5a5c119d5,43239,1731552190584 from backup master directory 2024-11-14T02:43:11,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41455-0x10138834f4e0001, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T02:43:11,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/83c5a5c119d5,43239,1731552190584 2024-11-14T02:43:11,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T02:43:11,412 WARN [master/83c5a5c119d5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T02:43:11,412 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=83c5a5c119d5,43239,1731552190584 2024-11-14T02:43:11,415 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-14T02:43:11,417 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-14T02:43:11,470 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/hbase.id] with ID: e010ce48-1cb1-4ad8-ae98-00527f341d4d 2024-11-14T02:43:11,470 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/.tmp/hbase.id 2024-11-14T02:43:11,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741826_1002 (size=42) 2024-11-14T02:43:11,886 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/.tmp/hbase.id]:[hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/hbase.id] 2024-11-14T02:43:11,937 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T02:43:11,942 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T02:43:11,960 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-14T02:43:11,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41455-0x10138834f4e0001, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:43:11,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:43:12,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741827_1003 (size=196) 2024-11-14T02:43:12,426 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T02:43:12,428 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T02:43:12,442 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:12,445 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T02:43:12,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741828_1004 (size=1189) 2024-11-14T02:43:12,902 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/MasterData/data/master/store 2024-11-14T02:43:12,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741829_1005 (size=34) 2024-11-14T02:43:13,332 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-14T02:43:13,336 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:43:13,337 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T02:43:13,337 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:43:13,338 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:43:13,339 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T02:43:13,340 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:43:13,340 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:43:13,341 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731552193337Disabling compacts and flushes for region at 1731552193337Disabling writes for close at 1731552193339 (+2 ms)Writing region close event to WAL at 1731552193340 (+1 ms)Closed at 1731552193340 2024-11-14T02:43:13,344 WARN [master/83c5a5c119d5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/MasterData/data/master/store/.initializing 2024-11-14T02:43:13,344 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/MasterData/WALs/83c5a5c119d5,43239,1731552190584 2024-11-14T02:43:13,351 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T02:43:13,364 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=83c5a5c119d5%2C43239%2C1731552190584, suffix=, logDir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/MasterData/WALs/83c5a5c119d5,43239,1731552190584, archiveDir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/MasterData/oldWALs, maxLogs=10 2024-11-14T02:43:13,383 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/MasterData/WALs/83c5a5c119d5,43239,1731552190584/83c5a5c119d5%2C43239%2C1731552190584.1731552193368, exclude list is [], retry=0 2024-11-14T02:43:13,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34599,DS-f7062220-044e-4c97-9b84-7d0dfdb6f9dc,DISK] 2024-11-14T02:43:13,400 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-14T02:43:13,433 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/MasterData/WALs/83c5a5c119d5,43239,1731552190584/83c5a5c119d5%2C43239%2C1731552190584.1731552193368 2024-11-14T02:43:13,434 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33069:33069)] 2024-11-14T02:43:13,435 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:43:13,435 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:43:13,439 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:43:13,440 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:43:13,474 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:43:13,497 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T02:43:13,501 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:13,503 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:43:13,504 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:43:13,508 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T02:43:13,508 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:13,509 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:43:13,509 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:43:13,512 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T02:43:13,513 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:13,513 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:43:13,514 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:43:13,517 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T02:43:13,517 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:13,518 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:43:13,518 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:43:13,522 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:43:13,523 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:43:13,528 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:43:13,529 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:43:13,533 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T02:43:13,537 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:43:13,542 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T02:43:13,543 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69528335, jitterRate=0.03605292737483978}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T02:43:13,550 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731552193451Initializing all the Stores at 1731552193453 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552193453Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552193454 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552193454Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552193454Cleaning up temporary data from old regions at 1731552193529 (+75 ms)Region opened successfully at 1731552193550 (+21 ms) 2024-11-14T02:43:13,551 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T02:43:13,580 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45186b19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=83c5a5c119d5/172.17.0.2:0 2024-11-14T02:43:13,605 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T02:43:13,615 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T02:43:13,616 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T02:43:13,618 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T02:43:13,620 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-14T02:43:13,624 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-14T02:43:13,624 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T02:43:13,649 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T02:43:13,660 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T02:43:13,685 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T02:43:13,689 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T02:43:13,691 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T02:43:13,702 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T02:43:13,705 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T02:43:13,712 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T02:43:13,718 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T02:43:13,720 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T02:43:13,727 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T02:43:13,747 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T02:43:13,752 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T02:43:13,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41455-0x10138834f4e0001, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T02:43:13,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T02:43:13,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41455-0x10138834f4e0001, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:43:13,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:43:13,764 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=83c5a5c119d5,43239,1731552190584, sessionid=0x10138834f4e0000, setting cluster-up flag (Was=false) 2024-11-14T02:43:13,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:43:13,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41455-0x10138834f4e0001, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:43:13,819 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T02:43:13,824 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=83c5a5c119d5,43239,1731552190584 2024-11-14T02:43:13,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41455-0x10138834f4e0001, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:43:13,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:43:13,869 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T02:43:13,873 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=83c5a5c119d5,43239,1731552190584 2024-11-14T02:43:13,882 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T02:43:13,947 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T02:43:13,956 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T02:43:13,957 INFO [RS:0;83c5a5c119d5:41455 {}] regionserver.HRegionServer(746): ClusterId : e010ce48-1cb1-4ad8-ae98-00527f341d4d 2024-11-14T02:43:13,959 DEBUG [RS:0;83c5a5c119d5:41455 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T02:43:13,962 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T02:43:13,967 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 83c5a5c119d5,43239,1731552190584 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T02:43:13,971 DEBUG [RS:0;83c5a5c119d5:41455 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T02:43:13,971 DEBUG [RS:0;83c5a5c119d5:41455 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T02:43:13,978 DEBUG [RS:0;83c5a5c119d5:41455 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T02:43:13,978 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/83c5a5c119d5:0, corePoolSize=5, maxPoolSize=5 2024-11-14T02:43:13,979 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/83c5a5c119d5:0, corePoolSize=5, maxPoolSize=5 2024-11-14T02:43:13,979 DEBUG [RS:0;83c5a5c119d5:41455 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@664c5e87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=83c5a5c119d5/172.17.0.2:0 2024-11-14T02:43:13,979 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/83c5a5c119d5:0, corePoolSize=5, maxPoolSize=5 2024-11-14T02:43:13,979 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/83c5a5c119d5:0, corePoolSize=5, maxPoolSize=5 2024-11-14T02:43:13,979 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/83c5a5c119d5:0, corePoolSize=10, maxPoolSize=10 2024-11-14T02:43:13,980 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/83c5a5c119d5:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:43:13,980 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/83c5a5c119d5:0, corePoolSize=2, maxPoolSize=2 2024-11-14T02:43:13,980 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/83c5a5c119d5:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:43:13,985 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731552223985 2024-11-14T02:43:13,986 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T02:43:13,986 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T02:43:13,987 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T02:43:13,989 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T02:43:13,993 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T02:43:13,994 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:13,994 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T02:43:13,994 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T02:43:13,994 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T02:43:13,995 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T02:43:13,995 DEBUG [RS:0;83c5a5c119d5:41455 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;83c5a5c119d5:41455 2024-11-14T02:43:13,995 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:13,999 INFO [RS:0;83c5a5c119d5:41455 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T02:43:14,000 INFO [RS:0;83c5a5c119d5:41455 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T02:43:14,000 DEBUG [RS:0;83c5a5c119d5:41455 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T02:43:14,001 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T02:43:14,002 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T02:43:14,002 INFO [RS:0;83c5a5c119d5:41455 {}] regionserver.HRegionServer(2659): reportForDuty to master=83c5a5c119d5,43239,1731552190584 with port=41455, startcode=1731552191259 2024-11-14T02:43:14,003 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T02:43:14,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741831_1007 (size=1321) 2024-11-14T02:43:14,005 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T02:43:14,005 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T02:43:14,007 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T02:43:14,008 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b 2024-11-14T02:43:14,014 DEBUG [RS:0;83c5a5c119d5:41455 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T02:43:14,015 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/83c5a5c119d5:0:becomeActiveMaster-HFileCleaner.large.0-1731552194007,5,FailOnTimeoutGroup] 2024-11-14T02:43:14,016 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/83c5a5c119d5:0:becomeActiveMaster-HFileCleaner.small.0-1731552194015,5,FailOnTimeoutGroup] 2024-11-14T02:43:14,016 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:14,017 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T02:43:14,018 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:14,018 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:14,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741832_1008 (size=32) 2024-11-14T02:43:14,051 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33791, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T02:43:14,057 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43239 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:14,059 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43239 {}] master.ServerManager(517): Registering regionserver=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:14,073 DEBUG [RS:0;83c5a5c119d5:41455 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b 2024-11-14T02:43:14,073 DEBUG [RS:0;83c5a5c119d5:41455 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37083 2024-11-14T02:43:14,073 DEBUG [RS:0;83c5a5c119d5:41455 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T02:43:14,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T02:43:14,086 DEBUG [RS:0;83c5a5c119d5:41455 {}] zookeeper.ZKUtil(111): regionserver:41455-0x10138834f4e0001, quorum=127.0.0.1:60414, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:14,087 WARN [RS:0;83c5a5c119d5:41455 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T02:43:14,087 INFO [RS:0;83c5a5c119d5:41455 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T02:43:14,087 DEBUG [RS:0;83c5a5c119d5:41455 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/WALs/83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:14,089 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [83c5a5c119d5,41455,1731552191259] 2024-11-14T02:43:14,114 INFO [RS:0;83c5a5c119d5:41455 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T02:43:14,129 INFO [RS:0;83c5a5c119d5:41455 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T02:43:14,134 INFO [RS:0;83c5a5c119d5:41455 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T02:43:14,134 INFO [RS:0;83c5a5c119d5:41455 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:14,135 INFO [RS:0;83c5a5c119d5:41455 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T02:43:14,140 INFO [RS:0;83c5a5c119d5:41455 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T02:43:14,141 INFO [RS:0;83c5a5c119d5:41455 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:14,142 DEBUG [RS:0;83c5a5c119d5:41455 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/83c5a5c119d5:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:43:14,142 DEBUG [RS:0;83c5a5c119d5:41455 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/83c5a5c119d5:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:43:14,142 DEBUG [RS:0;83c5a5c119d5:41455 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/83c5a5c119d5:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:43:14,142 DEBUG [RS:0;83c5a5c119d5:41455 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/83c5a5c119d5:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:43:14,142 DEBUG [RS:0;83c5a5c119d5:41455 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/83c5a5c119d5:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:43:14,142 DEBUG [RS:0;83c5a5c119d5:41455 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/83c5a5c119d5:0, corePoolSize=2, maxPoolSize=2 2024-11-14T02:43:14,143 DEBUG [RS:0;83c5a5c119d5:41455 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:43:14,143 DEBUG [RS:0;83c5a5c119d5:41455 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/83c5a5c119d5:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:43:14,143 DEBUG [RS:0;83c5a5c119d5:41455 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/83c5a5c119d5:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:43:14,143 DEBUG [RS:0;83c5a5c119d5:41455 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/83c5a5c119d5:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:43:14,143 DEBUG [RS:0;83c5a5c119d5:41455 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/83c5a5c119d5:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:43:14,143 DEBUG [RS:0;83c5a5c119d5:41455 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/83c5a5c119d5:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:43:14,143 DEBUG [RS:0;83c5a5c119d5:41455 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/83c5a5c119d5:0, corePoolSize=3, maxPoolSize=3 2024-11-14T02:43:14,144 DEBUG [RS:0;83c5a5c119d5:41455 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0, corePoolSize=3, maxPoolSize=3 2024-11-14T02:43:14,145 INFO [RS:0;83c5a5c119d5:41455 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:14,145 INFO [RS:0;83c5a5c119d5:41455 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:14,145 INFO [RS:0;83c5a5c119d5:41455 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:14,145 INFO [RS:0;83c5a5c119d5:41455 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:14,146 INFO [RS:0;83c5a5c119d5:41455 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:14,146 INFO [RS:0;83c5a5c119d5:41455 {}] hbase.ChoreService(168): Chore ScheduledChore name=83c5a5c119d5,41455,1731552191259-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T02:43:14,163 INFO [RS:0;83c5a5c119d5:41455 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T02:43:14,165 INFO [RS:0;83c5a5c119d5:41455 {}] hbase.ChoreService(168): Chore ScheduledChore name=83c5a5c119d5,41455,1731552191259-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:14,166 INFO [RS:0;83c5a5c119d5:41455 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:14,166 INFO [RS:0;83c5a5c119d5:41455 {}] regionserver.Replication(171): 83c5a5c119d5,41455,1731552191259 started 2024-11-14T02:43:14,183 INFO [RS:0;83c5a5c119d5:41455 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:14,183 INFO [RS:0;83c5a5c119d5:41455 {}] regionserver.HRegionServer(1482): Serving as 83c5a5c119d5,41455,1731552191259, RpcServer on 83c5a5c119d5/172.17.0.2:41455, sessionid=0x10138834f4e0001 2024-11-14T02:43:14,184 DEBUG [RS:0;83c5a5c119d5:41455 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T02:43:14,184 DEBUG [RS:0;83c5a5c119d5:41455 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:14,184 DEBUG [RS:0;83c5a5c119d5:41455 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '83c5a5c119d5,41455,1731552191259' 2024-11-14T02:43:14,184 DEBUG [RS:0;83c5a5c119d5:41455 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T02:43:14,185 DEBUG [RS:0;83c5a5c119d5:41455 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T02:43:14,186 DEBUG [RS:0;83c5a5c119d5:41455 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T02:43:14,186 DEBUG [RS:0;83c5a5c119d5:41455 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T02:43:14,186 DEBUG [RS:0;83c5a5c119d5:41455 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:14,186 DEBUG [RS:0;83c5a5c119d5:41455 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '83c5a5c119d5,41455,1731552191259' 2024-11-14T02:43:14,186 DEBUG [RS:0;83c5a5c119d5:41455 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T02:43:14,187 DEBUG [RS:0;83c5a5c119d5:41455 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T02:43:14,188 DEBUG [RS:0;83c5a5c119d5:41455 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T02:43:14,188 INFO [RS:0;83c5a5c119d5:41455 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T02:43:14,188 INFO [RS:0;83c5a5c119d5:41455 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T02:43:14,297 INFO [RS:0;83c5a5c119d5:41455 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T02:43:14,300 INFO [RS:0;83c5a5c119d5:41455 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=83c5a5c119d5%2C41455%2C1731552191259, suffix=, logDir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/WALs/83c5a5c119d5,41455,1731552191259, archiveDir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/oldWALs, maxLogs=32 2024-11-14T02:43:14,315 DEBUG [RS:0;83c5a5c119d5:41455 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/WALs/83c5a5c119d5,41455,1731552191259/83c5a5c119d5%2C41455%2C1731552191259.1731552194303, exclude list is [], retry=0 2024-11-14T02:43:14,320 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34599,DS-f7062220-044e-4c97-9b84-7d0dfdb6f9dc,DISK] 2024-11-14T02:43:14,324 INFO [RS:0;83c5a5c119d5:41455 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/WALs/83c5a5c119d5,41455,1731552191259/83c5a5c119d5%2C41455%2C1731552191259.1731552194303 2024-11-14T02:43:14,325 DEBUG [RS:0;83c5a5c119d5:41455 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33069:33069)] 2024-11-14T02:43:14,429 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:43:14,434 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T02:43:14,438 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T02:43:14,439 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:14,440 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:43:14,440 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T02:43:14,443 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T02:43:14,443 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:14,444 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:43:14,444 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T02:43:14,446 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T02:43:14,446 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:14,447 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:43:14,447 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T02:43:14,450 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T02:43:14,450 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:14,451 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:43:14,451 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T02:43:14,452 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/hbase/meta/1588230740 2024-11-14T02:43:14,453 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/hbase/meta/1588230740 2024-11-14T02:43:14,455 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T02:43:14,455 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T02:43:14,456 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T02:43:14,459 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T02:43:14,463 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T02:43:14,464 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73486192, jitterRate=0.09502959251403809}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T02:43:14,466 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731552194429Initializing all the Stores at 1731552194433 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552194433Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552194433Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552194434 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552194434Cleaning up temporary data from old regions at 1731552194456 (+22 ms)Region opened successfully at 1731552194466 (+10 ms) 2024-11-14T02:43:14,466 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T02:43:14,466 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T02:43:14,466 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T02:43:14,466 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T02:43:14,466 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T02:43:14,468 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T02:43:14,468 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731552194466Disabling compacts and flushes for region at 1731552194466Disabling writes for close at 1731552194466Writing region close event to WAL at 1731552194467 (+1 ms)Closed at 1731552194468 (+1 ms) 2024-11-14T02:43:14,471 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T02:43:14,471 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T02:43:14,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T02:43:14,484 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T02:43:14,486 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T02:43:14,642 DEBUG [83c5a5c119d5:43239 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-14T02:43:14,657 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:14,662 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 83c5a5c119d5,41455,1731552191259, state=OPENING 2024-11-14T02:43:14,694 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T02:43:14,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41455-0x10138834f4e0001, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:43:14,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:43:14,704 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T02:43:14,704 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T02:43:14,706 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T02:43:14,709 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=83c5a5c119d5,41455,1731552191259}] 2024-11-14T02:43:14,887 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T02:43:14,890 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35379, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T02:43:14,903 INFO [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T02:43:14,903 INFO [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T02:43:14,904 INFO [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T02:43:14,907 INFO [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=83c5a5c119d5%2C41455%2C1731552191259.meta, suffix=.meta, logDir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/WALs/83c5a5c119d5,41455,1731552191259, archiveDir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/oldWALs, maxLogs=32 2024-11-14T02:43:14,922 DEBUG [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/WALs/83c5a5c119d5,41455,1731552191259/83c5a5c119d5%2C41455%2C1731552191259.meta.1731552194909.meta, exclude list is [], retry=0 2024-11-14T02:43:14,926 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34599,DS-f7062220-044e-4c97-9b84-7d0dfdb6f9dc,DISK] 2024-11-14T02:43:14,929 INFO [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/WALs/83c5a5c119d5,41455,1731552191259/83c5a5c119d5%2C41455%2C1731552191259.meta.1731552194909.meta 2024-11-14T02:43:14,929 DEBUG [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33069:33069)] 2024-11-14T02:43:14,930 DEBUG [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:43:14,932 DEBUG [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T02:43:14,934 DEBUG [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T02:43:14,938 INFO [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T02:43:14,942 DEBUG [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T02:43:14,943 DEBUG [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:43:14,943 DEBUG [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T02:43:14,943 DEBUG [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T02:43:14,947 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T02:43:14,948 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T02:43:14,948 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:14,949 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:43:14,950 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T02:43:14,951 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T02:43:14,951 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:14,952 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:43:14,952 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T02:43:14,953 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T02:43:14,953 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:14,954 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:43:14,955 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T02:43:14,956 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T02:43:14,956 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:14,957 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:43:14,957 DEBUG [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T02:43:14,959 DEBUG [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/hbase/meta/1588230740 2024-11-14T02:43:14,962 DEBUG [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/hbase/meta/1588230740 2024-11-14T02:43:14,964 DEBUG [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T02:43:14,965 DEBUG [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T02:43:14,965 DEBUG [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T02:43:14,968 DEBUG [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T02:43:14,970 INFO [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74554215, jitterRate=0.11094437539577484}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T02:43:14,970 DEBUG [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T02:43:14,972 DEBUG [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731552194944Writing region info on filesystem at 1731552194944Initializing all the Stores at 1731552194946 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552194946Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552194946Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552194946Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552194946Cleaning up temporary data from old regions at 1731552194965 (+19 ms)Running coprocessor post-open hooks at 1731552194970 (+5 ms)Region opened successfully at 1731552194971 (+1 ms) 2024-11-14T02:43:14,978 INFO [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731552194877 2024-11-14T02:43:14,987 DEBUG [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T02:43:14,988 INFO [RS_OPEN_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T02:43:14,990 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:14,993 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 83c5a5c119d5,41455,1731552191259, state=OPEN 2024-11-14T02:43:15,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T02:43:15,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41455-0x10138834f4e0001, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T02:43:15,050 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T02:43:15,050 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T02:43:15,050 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:15,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T02:43:15,060 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=83c5a5c119d5,41455,1731552191259 in 341 msec 2024-11-14T02:43:15,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T02:43:15,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 585 msec 2024-11-14T02:43:15,069 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T02:43:15,069 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T02:43:15,087 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:15,089 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:43:15,109 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:15,112 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55855, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:15,131 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2210 sec 2024-11-14T02:43:15,132 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731552195132, completionTime=-1 2024-11-14T02:43:15,135 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-14T02:43:15,135 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T02:43:15,161 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-14T02:43:15,161 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731552255161 2024-11-14T02:43:15,161 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731552315161 2024-11-14T02:43:15,161 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 26 msec 2024-11-14T02:43:15,163 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=83c5a5c119d5,43239,1731552190584-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:15,163 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=83c5a5c119d5,43239,1731552190584-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:15,163 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=83c5a5c119d5,43239,1731552190584-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:15,165 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-83c5a5c119d5:43239, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:15,165 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:15,165 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:15,170 DEBUG [master/83c5a5c119d5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T02:43:15,190 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.778sec 2024-11-14T02:43:15,191 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T02:43:15,192 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T02:43:15,193 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T02:43:15,193 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T02:43:15,193 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T02:43:15,194 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=83c5a5c119d5,43239,1731552190584-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T02:43:15,195 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=83c5a5c119d5,43239,1731552190584-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T02:43:15,204 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T02:43:15,205 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T02:43:15,205 INFO [master/83c5a5c119d5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=83c5a5c119d5,43239,1731552190584-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:15,274 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:15,275 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-14T02:43:15,275 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-14T02:43:15,278 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:43:15,280 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:15,287 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:43:15,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:15,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:43:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@424b8b64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:43:15,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:15,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:15,297 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:15,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:15,300 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:15,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:43:15,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:15,309 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:15,331 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=83c5a5c119d5,43239,1731552190584 2024-11-14T02:43:15,341 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=208, ProcessCount=11, AvailableMemoryMB=6071 2024-11-14T02:43:15,368 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T02:43:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 83c5a5c119d5,43239,1731552190584 2024-11-14T02:43:15,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@27423f7d 2024-11-14T02:43:15,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T02:43:15,381 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T02:43:15,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43239 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T02:43:15,390 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43239 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T02:43:15,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T02:43:15,408 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T02:43:15,410 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43239 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-14T02:43:15,411 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:15,414 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T02:43:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T02:43:15,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741835_1011 (size=959) 2024-11-14T02:43:15,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T02:43:15,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T02:43:15,831 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b 2024-11-14T02:43:15,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741836_1012 (size=53) 2024-11-14T02:43:16,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T02:43:16,244 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:43:16,245 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 557913855dcf97d8b784fddf23bec5ee, disabling compactions & flushes 2024-11-14T02:43:16,245 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:16,245 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:16,246 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. after waiting 0 ms 2024-11-14T02:43:16,246 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:16,246 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:16,247 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 557913855dcf97d8b784fddf23bec5ee: Waiting for close lock at 1731552196245Disabling compacts and flushes for region at 1731552196245Disabling writes for close at 1731552196246 (+1 ms)Writing region close event to WAL at 1731552196246Closed at 1731552196246 2024-11-14T02:43:16,250 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T02:43:16,257 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731552196250"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731552196250"}]},"ts":"1731552196250"} 2024-11-14T02:43:16,261 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T02:43:16,263 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T02:43:16,266 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552196263"}]},"ts":"1731552196263"} 2024-11-14T02:43:16,270 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T02:43:16,271 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=557913855dcf97d8b784fddf23bec5ee, ASSIGN}] 2024-11-14T02:43:16,275 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=557913855dcf97d8b784fddf23bec5ee, ASSIGN 2024-11-14T02:43:16,277 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=557913855dcf97d8b784fddf23bec5ee, ASSIGN; state=OFFLINE, location=83c5a5c119d5,41455,1731552191259; forceNewPlan=false, retain=false 2024-11-14T02:43:16,429 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=557913855dcf97d8b784fddf23bec5ee, regionState=OPENING, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:16,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=557913855dcf97d8b784fddf23bec5ee, ASSIGN because future has completed 2024-11-14T02:43:16,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259}] 2024-11-14T02:43:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T02:43:16,605 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:16,605 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:43:16,606 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:16,606 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:43:16,606 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:16,606 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:16,609 INFO [StoreOpener-557913855dcf97d8b784fddf23bec5ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:16,611 INFO [StoreOpener-557913855dcf97d8b784fddf23bec5ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 557913855dcf97d8b784fddf23bec5ee columnFamilyName A 2024-11-14T02:43:16,611 DEBUG [StoreOpener-557913855dcf97d8b784fddf23bec5ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:16,612 INFO [StoreOpener-557913855dcf97d8b784fddf23bec5ee-1 {}] regionserver.HStore(327): Store=557913855dcf97d8b784fddf23bec5ee/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:43:16,613 INFO [StoreOpener-557913855dcf97d8b784fddf23bec5ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:16,615 INFO [StoreOpener-557913855dcf97d8b784fddf23bec5ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 557913855dcf97d8b784fddf23bec5ee columnFamilyName B 2024-11-14T02:43:16,615 DEBUG [StoreOpener-557913855dcf97d8b784fddf23bec5ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:16,616 INFO [StoreOpener-557913855dcf97d8b784fddf23bec5ee-1 {}] regionserver.HStore(327): Store=557913855dcf97d8b784fddf23bec5ee/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:43:16,616 INFO [StoreOpener-557913855dcf97d8b784fddf23bec5ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:16,618 INFO [StoreOpener-557913855dcf97d8b784fddf23bec5ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 557913855dcf97d8b784fddf23bec5ee columnFamilyName C 2024-11-14T02:43:16,618 DEBUG [StoreOpener-557913855dcf97d8b784fddf23bec5ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:16,619 INFO [StoreOpener-557913855dcf97d8b784fddf23bec5ee-1 {}] regionserver.HStore(327): Store=557913855dcf97d8b784fddf23bec5ee/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:43:16,619 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:16,621 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:16,622 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:16,624 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:16,624 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:16,625 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T02:43:16,628 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:16,632 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T02:43:16,633 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 557913855dcf97d8b784fddf23bec5ee; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73397068, jitterRate=0.09370154142379761}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T02:43:16,634 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:16,635 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 557913855dcf97d8b784fddf23bec5ee: Running coprocessor pre-open hook at 1731552196607Writing region info on filesystem at 1731552196607Initializing all the Stores at 1731552196608 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552196608Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552196609 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552196609Cleaning up temporary data from old regions at 1731552196624 (+15 ms)Running coprocessor post-open hooks at 1731552196634 (+10 ms)Region opened successfully at 1731552196635 (+1 ms) 2024-11-14T02:43:16,637 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., pid=6, masterSystemTime=1731552196595 2024-11-14T02:43:16,640 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:16,640 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:16,641 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=557913855dcf97d8b784fddf23bec5ee, regionState=OPEN, openSeqNum=2, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:16,645 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 because future has completed 2024-11-14T02:43:16,651 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T02:43:16,651 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 in 206 msec 2024-11-14T02:43:16,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T02:43:16,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=557913855dcf97d8b784fddf23bec5ee, ASSIGN in 380 msec 2024-11-14T02:43:16,657 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T02:43:16,657 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552196657"}]},"ts":"1731552196657"} 2024-11-14T02:43:16,661 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T02:43:16,662 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T02:43:16,666 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2700 sec 2024-11-14T02:43:17,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T02:43:17,588 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:17,597 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,597 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:43:17,597 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:17,598 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:43:17,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:17,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:43:17,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24219159, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:43:17,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:17,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:17,601 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:17,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b3feb14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,603 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:17,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:43:17,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:17,607 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:17,610 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T02:43:17,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 83c5a5c119d5,43239,1731552190584 2024-11-14T02:43:17,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6f451fdf 2024-11-14T02:43:17,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T02:43:17,616 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T02:43:17,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,620 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:43:17,620 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:17,622 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:43:17,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:17,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:43:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d21add1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:43:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:17,624 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:17,625 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:17,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d4406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,627 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,627 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:43:17,628 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:17,629 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:43:17,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:17,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:43:17,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@450047aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:43:17,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:17,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:17,634 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5676fb0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,636 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,636 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:43:17,636 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:17,638 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:43:17,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:17,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:43:17,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2544cc55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:43:17,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:17,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:17,641 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:17,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63181207, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,643 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:43:17,643 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:17,644 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:43:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:43:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d22a118, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:43:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:17,648 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8e0b13b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,650 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,650 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:43:17,650 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:17,651 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:43:17,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:17,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:43:17,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b21b95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:43:17,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:17,654 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:17,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a8d6bd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,656 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:43:17,656 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:17,657 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:43:17,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:17,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:43:17,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bc03052, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:43:17,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:17,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:17,660 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:17,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267419b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,662 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:43:17,662 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:17,663 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:43:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:43:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7788dec7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:43:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:17,666 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:17,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20ec50e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,668 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:43:17,668 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:17,670 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:43:17,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:17,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:43:17,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dc64f57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:43:17,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:17,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:17,672 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3adce01e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36df70c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,674 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:43:17,675 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:17,675 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:43:17,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:17,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:43:17,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12ce558b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:43:17,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:17,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:17,678 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:17,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e163b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:17,685 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:17,685 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:17,685 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:17,685 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:17,685 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:17,685 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:43:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:43:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:43:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:17,688 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:17,689 DEBUG [Thread-149 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:43:17,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:43:17,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:17,689 DEBUG [Thread-149 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:17,691 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:17,691 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:17,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:43:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:43:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:17,694 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:17,695 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:17,695 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:17,695 DEBUG [Thread-154 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:43:17,696 DEBUG [Thread-154 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:17,698 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:17,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:43:17,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:43:17,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:43:17,706 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:43:17,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:43:17,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:43:17,707 DEBUG [Thread-160 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:17,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:17,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:43:17,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:43:17,712 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:17,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-14T02:43:17,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:43:17,714 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:43:17,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T02:43:17,717 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:43:17,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:43:17,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:17,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:43:17,723 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:43:17,796 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:43:17,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T02:43:17,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:17,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T02:43:17,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T02:43:17,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:17,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:17,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:17,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:17,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/b881d7662bda4652885f29025da03180 is 50, key is test_row_0/A:col10/1731552197799/Put/seqid=0 2024-11-14T02:43:17,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:17,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741837_1013 (size=14341) 2024-11-14T02:43:17,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:17,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552257964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552257963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:17,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552257972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:17,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552257975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:17,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:17,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552257981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:18,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:18,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:18,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:18,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T02:43:18,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T02:43:18,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:18,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:18,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:18,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:18,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:18,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:18,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552258135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:18,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:18,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552258136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:18,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:18,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:18,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552258136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:18,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552258137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:18,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552258137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:18,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T02:43:18,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:18,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:18,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:18,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:18,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T02:43:18,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552258345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:18,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:18,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552258355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:18,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552258354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:18,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:18,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:18,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552258356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:18,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:18,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:18,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:18,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552258356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:18,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:18,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/b881d7662bda4652885f29025da03180 2024-11-14T02:43:18,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T02:43:18,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:18,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:18,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:18,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:18,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:18,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:18,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/aa2971cb61e64835982161764532ae6f is 50, key is test_row_0/B:col10/1731552197799/Put/seqid=0 2024-11-14T02:43:18,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741838_1014 (size=12001) 2024-11-14T02:43:18,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/aa2971cb61e64835982161764532ae6f 2024-11-14T02:43:18,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/ce1771b6fd704a11be22c322100ade66 is 50, key is test_row_0/C:col10/1731552197799/Put/seqid=0 2024-11-14T02:43:18,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741839_1015 (size=12001) 2024-11-14T02:43:18,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/ce1771b6fd704a11be22c322100ade66 2024-11-14T02:43:18,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T02:43:18,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:18,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:18,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:18,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:18,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:18,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/b881d7662bda4652885f29025da03180 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/b881d7662bda4652885f29025da03180 2024-11-14T02:43:18,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/b881d7662bda4652885f29025da03180, entries=200, sequenceid=14, filesize=14.0 K 2024-11-14T02:43:18,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/aa2971cb61e64835982161764532ae6f as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/aa2971cb61e64835982161764532ae6f 2024-11-14T02:43:18,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/aa2971cb61e64835982161764532ae6f, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T02:43:18,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/ce1771b6fd704a11be22c322100ade66 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/ce1771b6fd704a11be22c322100ade66 2024-11-14T02:43:18,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/ce1771b6fd704a11be22c322100ade66, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T02:43:18,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 557913855dcf97d8b784fddf23bec5ee in 864ms, sequenceid=14, compaction requested=false 2024-11-14T02:43:18,672 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T02:43:18,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:18,682 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:43:18,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T02:43:18,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/8c1ea73215ca4128ab404232f6363398 is 50, key is test_row_0/A:col10/1731552197870/Put/seqid=0 2024-11-14T02:43:18,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741840_1016 (size=16681) 2024-11-14T02:43:18,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:18,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/8c1ea73215ca4128ab404232f6363398 2024-11-14T02:43:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552258702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:18,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:18,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T02:43:18,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:18,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:18,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:18,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:18,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552258707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:18,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:18,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552258714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:18,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:18,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552258718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:18,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:18,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552258730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:18,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:18,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/de7e0c7e5ffd452f95bef8581834463c is 50, key is test_row_0/B:col10/1731552197870/Put/seqid=0 2024-11-14T02:43:18,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741841_1017 (size=12001) 2024-11-14T02:43:18,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/de7e0c7e5ffd452f95bef8581834463c 2024-11-14T02:43:18,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/66cd6f8a03ae4aa3a297499a23e9a05a is 50, key is test_row_0/C:col10/1731552197870/Put/seqid=0 2024-11-14T02:43:18,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552258834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:18,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:18,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552258844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:18,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:18,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552258844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:18,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:18,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552258845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:18,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T02:43:18,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552258854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:18,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741842_1018 (size=12001) 2024-11-14T02:43:18,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:18,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:18,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/66cd6f8a03ae4aa3a297499a23e9a05a 2024-11-14T02:43:18,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/8c1ea73215ca4128ab404232f6363398 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/8c1ea73215ca4128ab404232f6363398 2024-11-14T02:43:18,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T02:43:18,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:18,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:18,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:18,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:18,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:18,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/8c1ea73215ca4128ab404232f6363398, entries=250, sequenceid=40, filesize=16.3 K 2024-11-14T02:43:18,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/de7e0c7e5ffd452f95bef8581834463c as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/de7e0c7e5ffd452f95bef8581834463c 2024-11-14T02:43:18,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/de7e0c7e5ffd452f95bef8581834463c, entries=150, sequenceid=40, filesize=11.7 K 2024-11-14T02:43:18,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/66cd6f8a03ae4aa3a297499a23e9a05a as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/66cd6f8a03ae4aa3a297499a23e9a05a 2024-11-14T02:43:18,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/66cd6f8a03ae4aa3a297499a23e9a05a, entries=150, sequenceid=40, filesize=11.7 K 2024-11-14T02:43:18,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 557913855dcf97d8b784fddf23bec5ee in 268ms, sequenceid=40, compaction requested=false 2024-11-14T02:43:18,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:19,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T02:43:19,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:19,044 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-14T02:43:19,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:19,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/a8478e5d937049209a44185cd1a64c0b is 50, key is test_row_0/A:col10/1731552198725/Put/seqid=0 2024-11-14T02:43:19,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741843_1019 (size=9657) 2024-11-14T02:43:19,067 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/a8478e5d937049209a44185cd1a64c0b 2024-11-14T02:43:19,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/38203440bebd42d0b4d7299696c1e5e5 is 50, key is test_row_0/B:col10/1731552198725/Put/seqid=0 2024-11-14T02:43:19,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741844_1020 (size=9657) 2024-11-14T02:43:19,131 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/38203440bebd42d0b4d7299696c1e5e5 2024-11-14T02:43:19,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/c5e84dae5ace43dabae8f90fc58183bf is 50, key is test_row_0/C:col10/1731552198725/Put/seqid=0 2024-11-14T02:43:19,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741845_1021 (size=9657) 2024-11-14T02:43:19,193 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/c5e84dae5ace43dabae8f90fc58183bf 2024-11-14T02:43:19,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/a8478e5d937049209a44185cd1a64c0b as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/a8478e5d937049209a44185cd1a64c0b 2024-11-14T02:43:19,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:19,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552259201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:19,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:19,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:19,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552259206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:19,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:19,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552259211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:19,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:19,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:19,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552259215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:19,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:19,228 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/a8478e5d937049209a44185cd1a64c0b, entries=100, sequenceid=50, filesize=9.4 K 2024-11-14T02:43:19,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/38203440bebd42d0b4d7299696c1e5e5 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/38203440bebd42d0b4d7299696c1e5e5 2024-11-14T02:43:19,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:19,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552259219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:19,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:19,264 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/38203440bebd42d0b4d7299696c1e5e5, entries=100, sequenceid=50, filesize=9.4 K 2024-11-14T02:43:19,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/c5e84dae5ace43dabae8f90fc58183bf as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/c5e84dae5ace43dabae8f90fc58183bf 2024-11-14T02:43:19,280 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/c5e84dae5ace43dabae8f90fc58183bf, entries=100, sequenceid=50, filesize=9.4 K 2024-11-14T02:43:19,288 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 557913855dcf97d8b784fddf23bec5ee in 245ms, sequenceid=50, compaction requested=true 2024-11-14T02:43:19,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:19,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:19,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-14T02:43:19,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-14T02:43:19,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-14T02:43:19,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5730 sec 2024-11-14T02:43:19,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.5890 sec 2024-11-14T02:43:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:19,338 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:43:19,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-14T02:43:19,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/f33b7b57c76e4c80b8606a22bd59bfcf is 50, key is test_row_0/A:col10/1731552199213/Put/seqid=0 2024-11-14T02:43:19,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741846_1022 (size=14341) 2024-11-14T02:43:19,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/f33b7b57c76e4c80b8606a22bd59bfcf 2024-11-14T02:43:19,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:19,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552259376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:19,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:19,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552259378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:19,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:19,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/b79233acc8f341b682e819dc7eac0de9 is 50, key is test_row_0/B:col10/1731552199213/Put/seqid=0 2024-11-14T02:43:19,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:19,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552259382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:19,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:19,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:19,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552259385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:19,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:19,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741847_1023 (size=12001) 2024-11-14T02:43:19,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:19,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/b79233acc8f341b682e819dc7eac0de9 2024-11-14T02:43:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552259386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:19,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/b91b0ecbd17948ca944e9840a99f828b is 50, key is test_row_0/C:col10/1731552199213/Put/seqid=0 2024-11-14T02:43:19,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741848_1024 (size=12001) 2024-11-14T02:43:19,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/b91b0ecbd17948ca944e9840a99f828b 2024-11-14T02:43:19,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/f33b7b57c76e4c80b8606a22bd59bfcf as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f33b7b57c76e4c80b8606a22bd59bfcf 2024-11-14T02:43:19,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f33b7b57c76e4c80b8606a22bd59bfcf, entries=200, sequenceid=77, filesize=14.0 K 2024-11-14T02:43:19,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/b79233acc8f341b682e819dc7eac0de9 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/b79233acc8f341b682e819dc7eac0de9 2024-11-14T02:43:19,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/b79233acc8f341b682e819dc7eac0de9, entries=150, sequenceid=77, filesize=11.7 K 2024-11-14T02:43:19,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/b91b0ecbd17948ca944e9840a99f828b as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/b91b0ecbd17948ca944e9840a99f828b 2024-11-14T02:43:19,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:19,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552259494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:19,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:19,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552259494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:19,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:19,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:19,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/b91b0ecbd17948ca944e9840a99f828b, entries=150, sequenceid=77, filesize=11.7 K 2024-11-14T02:43:19,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 557913855dcf97d8b784fddf23bec5ee in 180ms, sequenceid=77, compaction requested=true 2024-11-14T02:43:19,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:19,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:43:19,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:19,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:43:19,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:19,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:43:19,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:19,526 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:43:19,528 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:43:19,534 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:43:19,536 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/B is initiating minor compaction (all files) 2024-11-14T02:43:19,536 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/B in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:19,537 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/aa2971cb61e64835982161764532ae6f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/de7e0c7e5ffd452f95bef8581834463c, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/38203440bebd42d0b4d7299696c1e5e5, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/b79233acc8f341b682e819dc7eac0de9] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=44.6 K 2024-11-14T02:43:19,540 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting aa2971cb61e64835982161764532ae6f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731552197774 2024-11-14T02:43:19,540 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55020 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:43:19,540 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/A is initiating minor compaction (all files) 2024-11-14T02:43:19,540 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/A in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:19,540 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/b881d7662bda4652885f29025da03180, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/8c1ea73215ca4128ab404232f6363398, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/a8478e5d937049209a44185cd1a64c0b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f33b7b57c76e4c80b8606a22bd59bfcf] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=53.7 K 2024-11-14T02:43:19,541 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting de7e0c7e5ffd452f95bef8581834463c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731552197870 2024-11-14T02:43:19,542 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting b881d7662bda4652885f29025da03180, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731552197774 2024-11-14T02:43:19,542 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 38203440bebd42d0b4d7299696c1e5e5, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731552198708 2024-11-14T02:43:19,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:19,543 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:43:19,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T02:43:19,547 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting b79233acc8f341b682e819dc7eac0de9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731552199209 2024-11-14T02:43:19,548 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c1ea73215ca4128ab404232f6363398, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731552197870 2024-11-14T02:43:19,549 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8478e5d937049209a44185cd1a64c0b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731552198708 2024-11-14T02:43:19,550 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting f33b7b57c76e4c80b8606a22bd59bfcf, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731552199205 2024-11-14T02:43:19,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/a31fd7a6d1e24bf4879c2e20d87c68a3 is 50, key is test_row_0/A:col10/1731552199538/Put/seqid=0 2024-11-14T02:43:19,615 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#B#compaction#14 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:19,617 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#A#compaction#13 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:19,618 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/6833848658a342dd957b7b509915b3d2 is 50, key is test_row_0/B:col10/1731552199213/Put/seqid=0 2024-11-14T02:43:19,620 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/d0f5fa4376d4495199278852d38706ea is 50, key is test_row_0/A:col10/1731552199213/Put/seqid=0 2024-11-14T02:43:19,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741849_1025 (size=16681) 2024-11-14T02:43:19,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741850_1026 (size=12139) 2024-11-14T02:43:19,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741851_1027 (size=12139) 2024-11-14T02:43:19,683 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/d0f5fa4376d4495199278852d38706ea as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/d0f5fa4376d4495199278852d38706ea 2024-11-14T02:43:19,713 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/A of 557913855dcf97d8b784fddf23bec5ee into d0f5fa4376d4495199278852d38706ea(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:19,713 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:19,716 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/A, priority=12, startTime=1731552199521; duration=0sec 2024-11-14T02:43:19,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:19,716 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:19,717 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:A 2024-11-14T02:43:19,717 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:43:19,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552259702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:19,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:19,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:19,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552259705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:19,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:19,721 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:43:19,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:19,722 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/C is initiating minor compaction (all files) 2024-11-14T02:43:19,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552259710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:19,722 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/C in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:19,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:19,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:19,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552259714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:19,723 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/ce1771b6fd704a11be22c322100ade66, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/66cd6f8a03ae4aa3a297499a23e9a05a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/c5e84dae5ace43dabae8f90fc58183bf, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/b91b0ecbd17948ca944e9840a99f828b] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=44.6 K 2024-11-14T02:43:19,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:19,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:19,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552259714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:19,724 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce1771b6fd704a11be22c322100ade66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731552197774 2024-11-14T02:43:19,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:19,726 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66cd6f8a03ae4aa3a297499a23e9a05a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731552197870 2024-11-14T02:43:19,727 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5e84dae5ace43dabae8f90fc58183bf, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731552198708 2024-11-14T02:43:19,729 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting b91b0ecbd17948ca944e9840a99f828b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731552199209 2024-11-14T02:43:19,758 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#C#compaction#15 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:19,759 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/a3147531e1044c0fbb91d087cf03d92b is 50, key is test_row_0/C:col10/1731552199213/Put/seqid=0 2024-11-14T02:43:19,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741852_1028 (size=12139) 2024-11-14T02:43:19,788 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/a3147531e1044c0fbb91d087cf03d92b as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/a3147531e1044c0fbb91d087cf03d92b 2024-11-14T02:43:19,803 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/C of 557913855dcf97d8b784fddf23bec5ee into a3147531e1044c0fbb91d087cf03d92b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:19,803 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:19,803 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/C, priority=12, startTime=1731552199524; duration=0sec 2024-11-14T02:43:19,803 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:19,803 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:C 2024-11-14T02:43:19,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:19,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552259824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:19,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552259824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:19,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:19,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:19,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552259834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:19,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T02:43:19,874 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:19,875 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:43:19,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-14T02:43:19,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T02:43:19,881 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:43:19,882 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:43:19,882 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:43:19,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T02:43:20,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/a31fd7a6d1e24bf4879c2e20d87c68a3 2024-11-14T02:43:20,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:20,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552260035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:20,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:20,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:20,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:20,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T02:43:20,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:20,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:20,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:20,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:20,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:20,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:20,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552260036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:20,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:20,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552260035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:20,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:20,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:20,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:20,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:20,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:20,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:20,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:20,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552260044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:20,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552260044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:20,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/700e119d2da04e71999e84c0f46194c8 is 50, key is test_row_0/B:col10/1731552199538/Put/seqid=0 2024-11-14T02:43:20,103 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/6833848658a342dd957b7b509915b3d2 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/6833848658a342dd957b7b509915b3d2 2024-11-14T02:43:20,117 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/B of 557913855dcf97d8b784fddf23bec5ee into 6833848658a342dd957b7b509915b3d2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:20,117 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:20,119 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/B, priority=12, startTime=1731552199524; duration=0sec 2024-11-14T02:43:20,120 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:20,120 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:B 2024-11-14T02:43:20,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741853_1029 (size=12001) 2024-11-14T02:43:20,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T02:43:20,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T02:43:20,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:20,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:20,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:20,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:20,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:20,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:20,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T02:43:20,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:20,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:20,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:20,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:20,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:20,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552260356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:20,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:20,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552260356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552260357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:20,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:20,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:20,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:20,455 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T02:43:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T02:43:20,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T02:43:20,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:20,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:20,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:20,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:20,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:20,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:20,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/700e119d2da04e71999e84c0f46194c8 2024-11-14T02:43:20,542 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T02:43:20,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:20,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552260544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:20,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:20,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:20,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:20,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/d31606dfc5d641c3a46489b17d64bc25 is 50, key is test_row_0/C:col10/1731552199538/Put/seqid=0 2024-11-14T02:43:20,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:20,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552260554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:20,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:20,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:20,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:20,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741854_1030 (size=12001) 2024-11-14T02:43:20,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/d31606dfc5d641c3a46489b17d64bc25 2024-11-14T02:43:20,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/a31fd7a6d1e24bf4879c2e20d87c68a3 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/a31fd7a6d1e24bf4879c2e20d87c68a3 2024-11-14T02:43:20,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/a31fd7a6d1e24bf4879c2e20d87c68a3, entries=250, sequenceid=90, filesize=16.3 K 2024-11-14T02:43:20,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/700e119d2da04e71999e84c0f46194c8 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/700e119d2da04e71999e84c0f46194c8 2024-11-14T02:43:20,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/700e119d2da04e71999e84c0f46194c8, entries=150, sequenceid=90, filesize=11.7 K 2024-11-14T02:43:20,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/d31606dfc5d641c3a46489b17d64bc25 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/d31606dfc5d641c3a46489b17d64bc25 2024-11-14T02:43:20,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/d31606dfc5d641c3a46489b17d64bc25, entries=150, sequenceid=90, filesize=11.7 K 2024-11-14T02:43:20,640 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 557913855dcf97d8b784fddf23bec5ee in 1097ms, sequenceid=90, compaction requested=false 2024-11-14T02:43:20,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:20,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T02:43:20,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:20,671 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-14T02:43:20,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/4a83b0abdc894935bcdf760cf016721d is 50, key is test_row_0/A:col10/1731552199704/Put/seqid=0 2024-11-14T02:43:20,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741855_1031 (size=12001) 2024-11-14T02:43:20,712 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/4a83b0abdc894935bcdf760cf016721d 2024-11-14T02:43:20,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/5c7662343f5144c3b1a0586e82916bb6 is 50, key is test_row_0/B:col10/1731552199704/Put/seqid=0 2024-11-14T02:43:20,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741856_1032 (size=12001) 2024-11-14T02:43:20,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:20,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:20,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:20,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552260899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:20,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:20,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:20,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:20,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552260900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:20,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552260904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:20,916 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T02:43:20,917 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-14T02:43:20,920 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T02:43:20,920 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-14T02:43:20,920 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-14T02:43:20,920 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-14T02:43:20,922 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T02:43:20,922 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-14T02:43:21,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T02:43:21,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552261013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:21,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:21,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552261014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:21,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:21,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552261014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:21,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:21,164 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/5c7662343f5144c3b1a0586e82916bb6 2024-11-14T02:43:21,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/81e55419b50c4660b7a81f16b9418bdd is 50, key is test_row_0/C:col10/1731552199704/Put/seqid=0 2024-11-14T02:43:21,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741857_1033 (size=12001) 2024-11-14T02:43:21,203 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/81e55419b50c4660b7a81f16b9418bdd 2024-11-14T02:43:21,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/4a83b0abdc894935bcdf760cf016721d as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/4a83b0abdc894935bcdf760cf016721d 2024-11-14T02:43:21,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:21,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552261223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:21,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552261223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:21,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:21,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552261224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:21,232 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/4a83b0abdc894935bcdf760cf016721d, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T02:43:21,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/5c7662343f5144c3b1a0586e82916bb6 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/5c7662343f5144c3b1a0586e82916bb6 2024-11-14T02:43:21,254 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/5c7662343f5144c3b1a0586e82916bb6, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T02:43:21,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/81e55419b50c4660b7a81f16b9418bdd as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/81e55419b50c4660b7a81f16b9418bdd 2024-11-14T02:43:21,273 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/81e55419b50c4660b7a81f16b9418bdd, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T02:43:21,275 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 557913855dcf97d8b784fddf23bec5ee in 604ms, sequenceid=116, compaction requested=true 2024-11-14T02:43:21,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:21,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:21,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-14T02:43:21,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-14T02:43:21,283 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-14T02:43:21,283 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3960 sec 2024-11-14T02:43:21,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.4100 sec 2024-11-14T02:43:21,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:21,538 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:43:21,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-14T02:43:21,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/689ee3865d3d4c0e8804409c49f607ac is 50, key is test_row_0/A:col10/1731552201537/Put/seqid=0 2024-11-14T02:43:21,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741858_1034 (size=16781) 2024-11-14T02:43:21,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:21,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552261589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:21,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:21,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:21,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552261589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:21,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:21,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:21,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552261590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:21,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:21,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:21,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552261593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:21,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:21,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552261598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:21,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:21,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:21,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552261705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:21,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:21,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552261705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:21,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:21,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552261706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:21,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:21,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552261706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:21,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:21,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:21,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:21,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552261713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:21,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:21,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:21,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552261914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:21,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:21,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552261914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:21,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552261915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:21,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:21,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552261915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:21,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:21,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:21,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:21,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:21,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552261924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:21,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:21,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:21,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/689ee3865d3d4c0e8804409c49f607ac 2024-11-14T02:43:22,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/9f7968b4061548279020bfe255363e99 is 50, key is test_row_0/B:col10/1731552201537/Put/seqid=0 2024-11-14T02:43:22,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741859_1035 (size=12101) 2024-11-14T02:43:22,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/9f7968b4061548279020bfe255363e99 2024-11-14T02:43:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T02:43:22,024 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:22,025 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:43:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-14T02:43:22,032 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:43:22,034 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:43:22,035 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:43:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T02:43:22,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/c919e4a563e34727923baa4c0d0ec9e5 is 50, key is test_row_0/C:col10/1731552201537/Put/seqid=0 2024-11-14T02:43:22,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741860_1036 (size=12101) 2024-11-14T02:43:22,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/c919e4a563e34727923baa4c0d0ec9e5 2024-11-14T02:43:22,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/689ee3865d3d4c0e8804409c49f607ac as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/689ee3865d3d4c0e8804409c49f607ac 2024-11-14T02:43:22,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/689ee3865d3d4c0e8804409c49f607ac, entries=250, sequenceid=131, filesize=16.4 K 2024-11-14T02:43:22,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/9f7968b4061548279020bfe255363e99 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/9f7968b4061548279020bfe255363e99 2024-11-14T02:43:22,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/9f7968b4061548279020bfe255363e99, entries=150, sequenceid=131, filesize=11.8 K 2024-11-14T02:43:22,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/c919e4a563e34727923baa4c0d0ec9e5 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/c919e4a563e34727923baa4c0d0ec9e5 2024-11-14T02:43:22,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/c919e4a563e34727923baa4c0d0ec9e5, entries=150, sequenceid=131, filesize=11.8 K 2024-11-14T02:43:22,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 557913855dcf97d8b784fddf23bec5ee in 590ms, sequenceid=131, compaction requested=true 2024-11-14T02:43:22,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:22,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:43:22,129 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:43:22,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:22,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:43:22,129 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:43:22,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:22,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:43:22,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:22,132 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:43:22,132 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/B is initiating minor compaction (all files) 2024-11-14T02:43:22,132 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/B in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:22,133 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/6833848658a342dd957b7b509915b3d2, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/700e119d2da04e71999e84c0f46194c8, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/5c7662343f5144c3b1a0586e82916bb6, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/9f7968b4061548279020bfe255363e99] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=47.1 K 2024-11-14T02:43:22,133 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 6833848658a342dd957b7b509915b3d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731552199209 2024-11-14T02:43:22,134 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 700e119d2da04e71999e84c0f46194c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731552199538 2024-11-14T02:43:22,135 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c7662343f5144c3b1a0586e82916bb6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731552199656 2024-11-14T02:43:22,135 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57602 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:43:22,136 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/A is initiating minor compaction (all files) 2024-11-14T02:43:22,136 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/A in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:22,136 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/d0f5fa4376d4495199278852d38706ea, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/a31fd7a6d1e24bf4879c2e20d87c68a3, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/4a83b0abdc894935bcdf760cf016721d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/689ee3865d3d4c0e8804409c49f607ac] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=56.3 K 2024-11-14T02:43:22,136 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f7968b4061548279020bfe255363e99, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731552200886 2024-11-14T02:43:22,137 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0f5fa4376d4495199278852d38706ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731552199209 2024-11-14T02:43:22,138 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting a31fd7a6d1e24bf4879c2e20d87c68a3, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731552199345 2024-11-14T02:43:22,139 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a83b0abdc894935bcdf760cf016721d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731552199656 2024-11-14T02:43:22,140 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 689ee3865d3d4c0e8804409c49f607ac, keycount=250, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731552200886 2024-11-14T02:43:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T02:43:22,169 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#B#compaction#25 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:22,170 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#A#compaction#24 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:22,170 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/d6678907e9094d0da4eb5a460dd881d4 is 50, key is test_row_0/B:col10/1731552201537/Put/seqid=0 2024-11-14T02:43:22,170 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/ecbb14a46f024f30b5d377b18b765d8d is 50, key is test_row_0/A:col10/1731552201537/Put/seqid=0 2024-11-14T02:43:22,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T02:43:22,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:22,192 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-14T02:43:22,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741861_1037 (size=12375) 2024-11-14T02:43:22,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741862_1038 (size=12375) 2024-11-14T02:43:22,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/e70c6ebda19a46439906e8595c565a21 is 50, key is test_row_0/A:col10/1731552201589/Put/seqid=0 2024-11-14T02:43:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:22,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:22,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741863_1039 (size=12151) 2024-11-14T02:43:22,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:22,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:22,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552262250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:22,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552262251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:22,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:22,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552262255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:22,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:22,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552262256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:22,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:22,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:22,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552262256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:22,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:22,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:22,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:22,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T02:43:22,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552262365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:22,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:22,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:22,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552262374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552262375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:22,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:22,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552262375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552262375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:22,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:22,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552262574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:22,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552262584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:22,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:22,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552262584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:22,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:22,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552262593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:22,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:22,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552262593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:22,610 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/ecbb14a46f024f30b5d377b18b765d8d as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/ecbb14a46f024f30b5d377b18b765d8d 2024-11-14T02:43:22,618 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/d6678907e9094d0da4eb5a460dd881d4 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/d6678907e9094d0da4eb5a460dd881d4 2024-11-14T02:43:22,628 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/A of 557913855dcf97d8b784fddf23bec5ee into ecbb14a46f024f30b5d377b18b765d8d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:22,628 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:22,628 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/A, priority=12, startTime=1731552202129; duration=0sec 2024-11-14T02:43:22,629 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:22,629 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:A 2024-11-14T02:43:22,629 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:43:22,633 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:43:22,633 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/C is initiating minor compaction (all files) 2024-11-14T02:43:22,633 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/C in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:22,634 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/a3147531e1044c0fbb91d087cf03d92b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/d31606dfc5d641c3a46489b17d64bc25, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/81e55419b50c4660b7a81f16b9418bdd, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/c919e4a563e34727923baa4c0d0ec9e5] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=47.1 K 2024-11-14T02:43:22,634 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3147531e1044c0fbb91d087cf03d92b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731552199209 2024-11-14T02:43:22,635 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting d31606dfc5d641c3a46489b17d64bc25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731552199538 2024-11-14T02:43:22,636 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81e55419b50c4660b7a81f16b9418bdd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731552199656 2024-11-14T02:43:22,637 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/B of 557913855dcf97d8b784fddf23bec5ee into d6678907e9094d0da4eb5a460dd881d4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:22,637 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:22,637 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/B, priority=12, startTime=1731552202129; duration=0sec 2024-11-14T02:43:22,637 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:22,637 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:B 2024-11-14T02:43:22,637 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting c919e4a563e34727923baa4c0d0ec9e5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731552200886 2024-11-14T02:43:22,646 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/e70c6ebda19a46439906e8595c565a21 2024-11-14T02:43:22,661 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#C#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:22,662 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/2a238c9b6d5d425fb593739c700e3128 is 50, key is test_row_0/C:col10/1731552201537/Put/seqid=0 2024-11-14T02:43:22,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/e943f2bf8f274d68af39a95a8290fb2c is 50, key is test_row_0/B:col10/1731552201589/Put/seqid=0 2024-11-14T02:43:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T02:43:22,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741865_1041 (size=12151) 2024-11-14T02:43:22,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741864_1040 (size=12375) 2024-11-14T02:43:22,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552262884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:22,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:22,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552262893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:22,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:22,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552262893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:22,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:22,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552262904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:22,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:22,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552262913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:22,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:22,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:23,112 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/e943f2bf8f274d68af39a95a8290fb2c 2024-11-14T02:43:23,127 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/2a238c9b6d5d425fb593739c700e3128 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/2a238c9b6d5d425fb593739c700e3128 2024-11-14T02:43:23,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/1f942a02e2e1497390cf3bfe427506a3 is 50, key is test_row_0/C:col10/1731552201589/Put/seqid=0 2024-11-14T02:43:23,141 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/C of 557913855dcf97d8b784fddf23bec5ee into 2a238c9b6d5d425fb593739c700e3128(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:23,141 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:23,141 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/C, priority=12, startTime=1731552202129; duration=0sec 2024-11-14T02:43:23,141 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:23,141 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:C 2024-11-14T02:43:23,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741866_1042 (size=12151) 2024-11-14T02:43:23,150 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/1f942a02e2e1497390cf3bfe427506a3 2024-11-14T02:43:23,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/e70c6ebda19a46439906e8595c565a21 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/e70c6ebda19a46439906e8595c565a21 2024-11-14T02:43:23,172 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/e70c6ebda19a46439906e8595c565a21, entries=150, sequenceid=152, filesize=11.9 K 2024-11-14T02:43:23,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T02:43:23,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/e943f2bf8f274d68af39a95a8290fb2c as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/e943f2bf8f274d68af39a95a8290fb2c 2024-11-14T02:43:23,184 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/e943f2bf8f274d68af39a95a8290fb2c, entries=150, sequenceid=152, filesize=11.9 K 2024-11-14T02:43:23,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/1f942a02e2e1497390cf3bfe427506a3 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/1f942a02e2e1497390cf3bfe427506a3 2024-11-14T02:43:23,199 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/1f942a02e2e1497390cf3bfe427506a3, entries=150, sequenceid=152, filesize=11.9 K 2024-11-14T02:43:23,201 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 557913855dcf97d8b784fddf23bec5ee in 1009ms, sequenceid=152, compaction requested=false 2024-11-14T02:43:23,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:23,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:23,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-14T02:43:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-14T02:43:23,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-14T02:43:23,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1690 sec 2024-11-14T02:43:23,211 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.1830 sec 2024-11-14T02:43:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:23,396 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:43:23,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-14T02:43:23,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/6a1d512972584f64a3aabc6fb84f26d5 is 50, key is test_row_0/A:col10/1731552202249/Put/seqid=0 2024-11-14T02:43:23,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741867_1043 (size=12151) 2024-11-14T02:43:23,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/6a1d512972584f64a3aabc6fb84f26d5 2024-11-14T02:43:23,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:23,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552263430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:23,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/5440d96c4de84723bf1b02ecc601e717 is 50, key is test_row_0/B:col10/1731552202249/Put/seqid=0 2024-11-14T02:43:23,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:23,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:23,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552263439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552263437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:23,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552263441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:23,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552263443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:23,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:23,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741868_1044 (size=12151) 2024-11-14T02:43:23,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:23,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552263553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552263554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:23,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552263554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552263554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:23,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:23,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:23,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552263554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:23,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:23,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552263763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:23,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:23,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552263764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:23,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:23,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:23,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552263764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552263764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:23,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:23,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:23,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552263764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:23,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:23,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:23,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/5440d96c4de84723bf1b02ecc601e717 2024-11-14T02:43:23,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/71929cc10d64459eac896ff23ee0d799 is 50, key is test_row_0/C:col10/1731552202249/Put/seqid=0 2024-11-14T02:43:23,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741869_1045 (size=12151) 2024-11-14T02:43:24,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552264073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:24,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:24,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552264073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:24,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:24,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552264073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:24,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:24,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552264073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:24,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:24,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552264074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:24,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:24,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T02:43:24,193 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:24,194 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:43:24,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-14T02:43:24,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T02:43:24,196 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:43:24,198 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:43:24,198 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:43:24,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/71929cc10d64459eac896ff23ee0d799 2024-11-14T02:43:24,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/6a1d512972584f64a3aabc6fb84f26d5 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/6a1d512972584f64a3aabc6fb84f26d5 2024-11-14T02:43:24,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/6a1d512972584f64a3aabc6fb84f26d5, entries=150, sequenceid=172, filesize=11.9 K 2024-11-14T02:43:24,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/5440d96c4de84723bf1b02ecc601e717 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/5440d96c4de84723bf1b02ecc601e717 2024-11-14T02:43:24,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T02:43:24,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/5440d96c4de84723bf1b02ecc601e717, entries=150, sequenceid=172, filesize=11.9 K 2024-11-14T02:43:24,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/71929cc10d64459eac896ff23ee0d799 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/71929cc10d64459eac896ff23ee0d799 2024-11-14T02:43:24,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/71929cc10d64459eac896ff23ee0d799, entries=150, sequenceid=172, filesize=11.9 K 2024-11-14T02:43:24,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 557913855dcf97d8b784fddf23bec5ee in 920ms, sequenceid=172, compaction requested=true 2024-11-14T02:43:24,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:24,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:43:24,316 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:24,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:24,317 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:24,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:43:24,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:24,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:43:24,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:24,320 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:24,320 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/A is initiating minor compaction (all files) 2024-11-14T02:43:24,321 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/A in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:24,321 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/ecbb14a46f024f30b5d377b18b765d8d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/e70c6ebda19a46439906e8595c565a21, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/6a1d512972584f64a3aabc6fb84f26d5] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=35.8 K 2024-11-14T02:43:24,322 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:24,322 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecbb14a46f024f30b5d377b18b765d8d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731552200886 2024-11-14T02:43:24,322 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/B is initiating minor compaction (all files) 2024-11-14T02:43:24,322 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/B in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:24,323 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/d6678907e9094d0da4eb5a460dd881d4, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/e943f2bf8f274d68af39a95a8290fb2c, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/5440d96c4de84723bf1b02ecc601e717] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=35.8 K 2024-11-14T02:43:24,323 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting e70c6ebda19a46439906e8595c565a21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731552201585 2024-11-14T02:43:24,323 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting d6678907e9094d0da4eb5a460dd881d4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731552200886 2024-11-14T02:43:24,324 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting e943f2bf8f274d68af39a95a8290fb2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731552201585 2024-11-14T02:43:24,324 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 5440d96c4de84723bf1b02ecc601e717, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731552202238 2024-11-14T02:43:24,325 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a1d512972584f64a3aabc6fb84f26d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731552202238 2024-11-14T02:43:24,342 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#A#compaction#33 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:24,342 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#B#compaction#34 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:24,343 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/2d99d9085c02429997fe03c428eb901f is 50, key is test_row_0/A:col10/1731552202249/Put/seqid=0 2024-11-14T02:43:24,343 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/11f3d24c6daf48fda91172039dcc9af0 is 50, key is test_row_0/B:col10/1731552202249/Put/seqid=0 2024-11-14T02:43:24,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741870_1046 (size=12527) 2024-11-14T02:43:24,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T02:43:24,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:24,355 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-14T02:43:24,373 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/2d99d9085c02429997fe03c428eb901f as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/2d99d9085c02429997fe03c428eb901f 2024-11-14T02:43:24,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741871_1047 (size=12527) 2024-11-14T02:43:24,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/d95a4005cbe74dc8a5851275c9004eaa is 50, key is test_row_0/A:col10/1731552203406/Put/seqid=0 2024-11-14T02:43:24,390 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/11f3d24c6daf48fda91172039dcc9af0 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/11f3d24c6daf48fda91172039dcc9af0 2024-11-14T02:43:24,390 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/A of 557913855dcf97d8b784fddf23bec5ee into 2d99d9085c02429997fe03c428eb901f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:24,390 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:24,390 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/A, priority=13, startTime=1731552204316; duration=0sec 2024-11-14T02:43:24,390 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:24,390 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:A 2024-11-14T02:43:24,390 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:24,394 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:24,394 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/C is initiating minor compaction (all files) 2024-11-14T02:43:24,394 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/C in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:24,395 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/2a238c9b6d5d425fb593739c700e3128, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/1f942a02e2e1497390cf3bfe427506a3, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/71929cc10d64459eac896ff23ee0d799] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=35.8 K 2024-11-14T02:43:24,395 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a238c9b6d5d425fb593739c700e3128, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731552200886 2024-11-14T02:43:24,396 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f942a02e2e1497390cf3bfe427506a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731552201585 2024-11-14T02:43:24,396 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71929cc10d64459eac896ff23ee0d799, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731552202238 2024-11-14T02:43:24,400 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/B of 557913855dcf97d8b784fddf23bec5ee into 11f3d24c6daf48fda91172039dcc9af0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:24,400 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:24,400 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/B, priority=13, startTime=1731552204316; duration=0sec 2024-11-14T02:43:24,401 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:24,401 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:B 2024-11-14T02:43:24,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741872_1048 (size=12151) 2024-11-14T02:43:24,413 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#C#compaction#36 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:24,414 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/dd3ffb033cb34e8ca902e6973686608a is 50, key is test_row_0/C:col10/1731552202249/Put/seqid=0 2024-11-14T02:43:24,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741873_1049 (size=12527) 2024-11-14T02:43:24,433 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/dd3ffb033cb34e8ca902e6973686608a as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/dd3ffb033cb34e8ca902e6973686608a 2024-11-14T02:43:24,445 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/C of 557913855dcf97d8b784fddf23bec5ee into dd3ffb033cb34e8ca902e6973686608a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:24,445 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:24,445 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/C, priority=13, startTime=1731552204318; duration=0sec 2024-11-14T02:43:24,445 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:24,445 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:C 2024-11-14T02:43:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T02:43:24,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:24,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552264602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:24,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:24,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552264606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:24,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552264607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:24,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552264608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:24,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:24,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552264609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:24,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:24,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552264713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:24,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:24,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552264723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:24,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552264723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:24,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:24,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552264724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552264724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:24,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:24,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:24,804 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/d95a4005cbe74dc8a5851275c9004eaa 2024-11-14T02:43:24,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/0ffd665bd244460ea53b05d45a43b81c is 50, key is test_row_0/B:col10/1731552203406/Put/seqid=0 2024-11-14T02:43:24,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741874_1050 (size=12151) 2024-11-14T02:43:24,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T02:43:24,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:24,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552264923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:24,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:24,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552264933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:24,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:24,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552264933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:24,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:24,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552264934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:24,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:24,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552264934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:24,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:24,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:25,220 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/0ffd665bd244460ea53b05d45a43b81c 2024-11-14T02:43:25,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:25,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552265233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:25,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:25,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552265243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:25,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552265244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:25,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:25,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:25,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552265244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:25,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:25,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:25,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552265253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:25,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:25,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/14bee822d1c5451bba526caf841fd39b is 50, key is test_row_0/C:col10/1731552203406/Put/seqid=0 2024-11-14T02:43:25,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741875_1051 (size=12151) 2024-11-14T02:43:25,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T02:43:25,680 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/14bee822d1c5451bba526caf841fd39b 2024-11-14T02:43:25,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/d95a4005cbe74dc8a5851275c9004eaa as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/d95a4005cbe74dc8a5851275c9004eaa 2024-11-14T02:43:25,694 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/d95a4005cbe74dc8a5851275c9004eaa, entries=150, sequenceid=191, filesize=11.9 K 2024-11-14T02:43:25,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/0ffd665bd244460ea53b05d45a43b81c as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/0ffd665bd244460ea53b05d45a43b81c 2024-11-14T02:43:25,703 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/0ffd665bd244460ea53b05d45a43b81c, entries=150, sequenceid=191, filesize=11.9 K 2024-11-14T02:43:25,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/14bee822d1c5451bba526caf841fd39b as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/14bee822d1c5451bba526caf841fd39b 2024-11-14T02:43:25,714 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/14bee822d1c5451bba526caf841fd39b, entries=150, sequenceid=191, filesize=11.9 K 2024-11-14T02:43:25,715 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 557913855dcf97d8b784fddf23bec5ee in 1361ms, sequenceid=191, compaction requested=false 2024-11-14T02:43:25,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:25,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:25,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-14T02:43:25,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-14T02:43:25,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-14T02:43:25,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5190 sec 2024-11-14T02:43:25,722 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.5260 sec 2024-11-14T02:43:25,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:25,745 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:43:25,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-14T02:43:25,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/342e5879b69445678c29ed313706266c is 50, key is test_row_0/A:col10/1731552204607/Put/seqid=0 2024-11-14T02:43:25,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741876_1052 (size=12151) 2024-11-14T02:43:25,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:25,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552265775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:25,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552265776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:25,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552265778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552265779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:25,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552265780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:25,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:25,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552265883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:25,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:25,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552265883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:25,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:25,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:25,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:25,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552265893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:25,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:25,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:25,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552265894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:25,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:25,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:25,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552265894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:25,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:25,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:26,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552266093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:26,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:26,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552266094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:26,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:26,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552266103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552266103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:26,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552266103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:26,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:26,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/342e5879b69445678c29ed313706266c 2024-11-14T02:43:26,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/6eb22ed252134022a96196026f655742 is 50, key is test_row_0/B:col10/1731552204607/Put/seqid=0 2024-11-14T02:43:26,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741877_1053 (size=12151) 2024-11-14T02:43:26,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/6eb22ed252134022a96196026f655742 2024-11-14T02:43:26,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/ddfb02a09b8e4fbaa8da6ab98c65a38c is 50, key is test_row_0/C:col10/1731552204607/Put/seqid=0 2024-11-14T02:43:26,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741878_1054 (size=12151) 2024-11-14T02:43:26,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/ddfb02a09b8e4fbaa8da6ab98c65a38c 2024-11-14T02:43:26,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/342e5879b69445678c29ed313706266c as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/342e5879b69445678c29ed313706266c 2024-11-14T02:43:26,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/342e5879b69445678c29ed313706266c, entries=150, sequenceid=213, filesize=11.9 K 2024-11-14T02:43:26,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/6eb22ed252134022a96196026f655742 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/6eb22ed252134022a96196026f655742 2024-11-14T02:43:26,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/6eb22ed252134022a96196026f655742, entries=150, sequenceid=213, filesize=11.9 K 2024-11-14T02:43:26,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/ddfb02a09b8e4fbaa8da6ab98c65a38c as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/ddfb02a09b8e4fbaa8da6ab98c65a38c 2024-11-14T02:43:26,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/ddfb02a09b8e4fbaa8da6ab98c65a38c, entries=150, sequenceid=213, filesize=11.9 K 2024-11-14T02:43:26,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 557913855dcf97d8b784fddf23bec5ee in 530ms, sequenceid=213, compaction requested=true 2024-11-14T02:43:26,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:26,276 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:26,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:43:26,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:26,276 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:26,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:43:26,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:26,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:43:26,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:26,278 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:26,278 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:26,278 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/A is initiating minor compaction (all files) 2024-11-14T02:43:26,278 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/B is initiating minor compaction (all files) 2024-11-14T02:43:26,278 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/A in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:26,278 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/B in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:26,278 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/11f3d24c6daf48fda91172039dcc9af0, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/0ffd665bd244460ea53b05d45a43b81c, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/6eb22ed252134022a96196026f655742] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=36.0 K 2024-11-14T02:43:26,278 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/2d99d9085c02429997fe03c428eb901f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/d95a4005cbe74dc8a5851275c9004eaa, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/342e5879b69445678c29ed313706266c] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=36.0 K 2024-11-14T02:43:26,279 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 11f3d24c6daf48fda91172039dcc9af0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731552202238 2024-11-14T02:43:26,280 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d99d9085c02429997fe03c428eb901f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731552202238 2024-11-14T02:43:26,280 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ffd665bd244460ea53b05d45a43b81c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731552203406 2024-11-14T02:43:26,281 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting d95a4005cbe74dc8a5851275c9004eaa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731552203406 2024-11-14T02:43:26,281 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eb22ed252134022a96196026f655742, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731552204604 2024-11-14T02:43:26,281 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 342e5879b69445678c29ed313706266c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731552204604 2024-11-14T02:43:26,298 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#A#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:26,300 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/8f9f964550bf4f8196361e4c217f1010 is 50, key is test_row_0/A:col10/1731552204607/Put/seqid=0 2024-11-14T02:43:26,303 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#B#compaction#43 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:26,304 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/736054f07d3948e8af6979849d588d5b is 50, key is test_row_0/B:col10/1731552204607/Put/seqid=0 2024-11-14T02:43:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T02:43:26,344 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:26,345 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:43:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-14T02:43:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T02:43:26,349 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:43:26,350 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:43:26,350 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:43:26,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741880_1056 (size=12629) 2024-11-14T02:43:26,364 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/736054f07d3948e8af6979849d588d5b as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/736054f07d3948e8af6979849d588d5b 2024-11-14T02:43:26,375 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/B of 557913855dcf97d8b784fddf23bec5ee into 736054f07d3948e8af6979849d588d5b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:26,376 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:26,376 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/B, priority=13, startTime=1731552206276; duration=0sec 2024-11-14T02:43:26,376 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:26,376 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:B 2024-11-14T02:43:26,376 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:26,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741879_1055 (size=12629) 2024-11-14T02:43:26,378 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:26,378 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/C is initiating minor compaction (all files) 2024-11-14T02:43:26,378 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/C in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:26,379 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/dd3ffb033cb34e8ca902e6973686608a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/14bee822d1c5451bba526caf841fd39b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/ddfb02a09b8e4fbaa8da6ab98c65a38c] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=36.0 K 2024-11-14T02:43:26,380 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting dd3ffb033cb34e8ca902e6973686608a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731552202238 2024-11-14T02:43:26,381 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 14bee822d1c5451bba526caf841fd39b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731552203406 2024-11-14T02:43:26,384 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting ddfb02a09b8e4fbaa8da6ab98c65a38c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731552204604 2024-11-14T02:43:26,390 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/8f9f964550bf4f8196361e4c217f1010 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/8f9f964550bf4f8196361e4c217f1010 2024-11-14T02:43:26,399 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/A of 557913855dcf97d8b784fddf23bec5ee into 8f9f964550bf4f8196361e4c217f1010(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:26,399 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:26,399 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/A, priority=13, startTime=1731552206276; duration=0sec 2024-11-14T02:43:26,399 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:26,399 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:A 2024-11-14T02:43:26,403 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#C#compaction#44 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:26,403 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/a6bde6ec23e34293bcb7e9d980319977 is 50, key is test_row_0/C:col10/1731552204607/Put/seqid=0 2024-11-14T02:43:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:26,408 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:43:26,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-14T02:43:26,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/85d102cc8b144925ab5073c7cdf6f3e5 is 50, key is test_row_0/A:col10/1731552205777/Put/seqid=0 2024-11-14T02:43:26,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552266433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:26,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:26,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552266434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:26,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:26,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552266437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:26,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:26,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552266439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:26,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552266440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:26,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:26,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:26,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741882_1058 (size=14541) 2024-11-14T02:43:26,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/85d102cc8b144925ab5073c7cdf6f3e5 2024-11-14T02:43:26,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741881_1057 (size=12629) 2024-11-14T02:43:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T02:43:26,464 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/a6bde6ec23e34293bcb7e9d980319977 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/a6bde6ec23e34293bcb7e9d980319977 2024-11-14T02:43:26,474 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/C of 557913855dcf97d8b784fddf23bec5ee into a6bde6ec23e34293bcb7e9d980319977(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:26,474 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:26,474 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/C, priority=13, startTime=1731552206277; duration=0sec 2024-11-14T02:43:26,475 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:26,475 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:C 2024-11-14T02:43:26,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/5c0047a241f64faabd1368754ecbba3b is 50, key is test_row_0/B:col10/1731552205777/Put/seqid=0 2024-11-14T02:43:26,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741883_1059 (size=12151) 2024-11-14T02:43:26,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/5c0047a241f64faabd1368754ecbba3b 2024-11-14T02:43:26,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T02:43:26,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:26,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:26,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:26,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:26,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:26,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/50b8cc0ae9d44dc9a8511846ffe4c8a7 is 50, key is test_row_0/C:col10/1731552205777/Put/seqid=0 2024-11-14T02:43:26,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552266544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:26,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:26,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741884_1060 (size=12151) 2024-11-14T02:43:26,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/50b8cc0ae9d44dc9a8511846ffe4c8a7 2024-11-14T02:43:26,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552266554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:26,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:26,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552266554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:26,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552266555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:26,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:26,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:26,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552266555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:26,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:26,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/85d102cc8b144925ab5073c7cdf6f3e5 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/85d102cc8b144925ab5073c7cdf6f3e5 2024-11-14T02:43:26,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/85d102cc8b144925ab5073c7cdf6f3e5, entries=200, sequenceid=233, filesize=14.2 K 2024-11-14T02:43:26,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/5c0047a241f64faabd1368754ecbba3b as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/5c0047a241f64faabd1368754ecbba3b 2024-11-14T02:43:26,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/5c0047a241f64faabd1368754ecbba3b, entries=150, sequenceid=233, filesize=11.9 K 2024-11-14T02:43:26,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/50b8cc0ae9d44dc9a8511846ffe4c8a7 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/50b8cc0ae9d44dc9a8511846ffe4c8a7 2024-11-14T02:43:26,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/50b8cc0ae9d44dc9a8511846ffe4c8a7, entries=150, sequenceid=233, filesize=11.9 K 2024-11-14T02:43:26,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 557913855dcf97d8b784fddf23bec5ee in 203ms, sequenceid=233, compaction requested=false 2024-11-14T02:43:26,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:26,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T02:43:26,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T02:43:26,663 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-14T02:43:26,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/5844f43649c94ac9a5d28a123021f99a is 50, key is test_row_0/A:col10/1731552206436/Put/seqid=0 2024-11-14T02:43:26,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741885_1061 (size=12151) 2024-11-14T02:43:26,680 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/5844f43649c94ac9a5d28a123021f99a 2024-11-14T02:43:26,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/402372cf6bfa47a0a724d19ea9a70bd0 is 50, key is test_row_0/B:col10/1731552206436/Put/seqid=0 2024-11-14T02:43:26,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741886_1062 (size=12151) 2024-11-14T02:43:26,710 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/402372cf6bfa47a0a724d19ea9a70bd0 2024-11-14T02:43:26,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/529d6cd9e5e14bf4a256a0b92d20a713 is 50, key is test_row_0/C:col10/1731552206436/Put/seqid=0 2024-11-14T02:43:26,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741887_1063 (size=12151) 2024-11-14T02:43:26,737 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/529d6cd9e5e14bf4a256a0b92d20a713 2024-11-14T02:43:26,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/5844f43649c94ac9a5d28a123021f99a as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/5844f43649c94ac9a5d28a123021f99a 2024-11-14T02:43:26,756 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/5844f43649c94ac9a5d28a123021f99a, entries=150, sequenceid=252, filesize=11.9 K 2024-11-14T02:43:26,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:26,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:26,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/402372cf6bfa47a0a724d19ea9a70bd0 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/402372cf6bfa47a0a724d19ea9a70bd0 2024-11-14T02:43:26,776 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/402372cf6bfa47a0a724d19ea9a70bd0, entries=150, sequenceid=252, filesize=11.9 K 2024-11-14T02:43:26,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/529d6cd9e5e14bf4a256a0b92d20a713 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/529d6cd9e5e14bf4a256a0b92d20a713 2024-11-14T02:43:26,794 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/529d6cd9e5e14bf4a256a0b92d20a713, entries=150, sequenceid=252, filesize=11.9 K 2024-11-14T02:43:26,799 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=40.25 KB/41220 for 557913855dcf97d8b784fddf23bec5ee in 136ms, sequenceid=252, compaction requested=true 2024-11-14T02:43:26,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:26,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:26,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-14T02:43:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-14T02:43:26,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-14T02:43:26,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 452 msec 2024-11-14T02:43:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:26,805 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:43:26,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T02:43:26,810 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 462 msec 2024-11-14T02:43:26,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/5bf45cd31ee348969942efd1246bcfbe is 50, key is test_row_0/A:col10/1731552206803/Put/seqid=0 2024-11-14T02:43:26,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741888_1064 (size=19521) 2024-11-14T02:43:26,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/5bf45cd31ee348969942efd1246bcfbe 2024-11-14T02:43:26,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/7bf087ea8c0e4a9cb5d72c60a800b056 is 50, key is test_row_0/B:col10/1731552206803/Put/seqid=0 2024-11-14T02:43:26,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741889_1065 (size=12251) 2024-11-14T02:43:26,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/7bf087ea8c0e4a9cb5d72c60a800b056 2024-11-14T02:43:26,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/26d87dcf48a548699e162a29efe0c994 is 50, key is test_row_0/C:col10/1731552206803/Put/seqid=0 2024-11-14T02:43:26,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552266892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:26,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:26,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552266894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:26,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:26,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741890_1066 (size=12251) 2024-11-14T02:43:26,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552266897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:26,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:26,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552266899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:26,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:26,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:26,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552266901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:26,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:26,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:26,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/26d87dcf48a548699e162a29efe0c994 2024-11-14T02:43:26,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/5bf45cd31ee348969942efd1246bcfbe as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/5bf45cd31ee348969942efd1246bcfbe 2024-11-14T02:43:26,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/5bf45cd31ee348969942efd1246bcfbe, entries=300, sequenceid=263, filesize=19.1 K 2024-11-14T02:43:26,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/7bf087ea8c0e4a9cb5d72c60a800b056 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/7bf087ea8c0e4a9cb5d72c60a800b056 2024-11-14T02:43:26,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/7bf087ea8c0e4a9cb5d72c60a800b056, entries=150, sequenceid=263, filesize=12.0 K 2024-11-14T02:43:26,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/26d87dcf48a548699e162a29efe0c994 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/26d87dcf48a548699e162a29efe0c994 2024-11-14T02:43:26,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/26d87dcf48a548699e162a29efe0c994, entries=150, sequenceid=263, filesize=12.0 K 2024-11-14T02:43:26,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 557913855dcf97d8b784fddf23bec5ee in 144ms, sequenceid=263, compaction requested=true 2024-11-14T02:43:26,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:26,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:43:26,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:26,949 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:43:26,949 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:43:26,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:43:26,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:26,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:43:26,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:26,951 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 58842 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:43:26,951 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49182 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:43:26,951 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/A is initiating minor compaction (all files) 2024-11-14T02:43:26,951 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/B is initiating minor compaction (all files) 2024-11-14T02:43:26,951 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/A in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:26,951 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/B in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:26,952 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/8f9f964550bf4f8196361e4c217f1010, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/85d102cc8b144925ab5073c7cdf6f3e5, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/5844f43649c94ac9a5d28a123021f99a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/5bf45cd31ee348969942efd1246bcfbe] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=57.5 K 2024-11-14T02:43:26,952 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/736054f07d3948e8af6979849d588d5b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/5c0047a241f64faabd1368754ecbba3b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/402372cf6bfa47a0a724d19ea9a70bd0, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/7bf087ea8c0e4a9cb5d72c60a800b056] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=48.0 K 2024-11-14T02:43:26,952 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f9f964550bf4f8196361e4c217f1010, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731552204604 2024-11-14T02:43:26,952 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 736054f07d3948e8af6979849d588d5b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731552204604 2024-11-14T02:43:26,953 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85d102cc8b144925ab5073c7cdf6f3e5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731552205773 2024-11-14T02:43:26,953 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c0047a241f64faabd1368754ecbba3b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731552205775 2024-11-14T02:43:26,954 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5844f43649c94ac9a5d28a123021f99a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731552206434 2024-11-14T02:43:26,955 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 402372cf6bfa47a0a724d19ea9a70bd0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731552206434 2024-11-14T02:43:26,955 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bf45cd31ee348969942efd1246bcfbe, keycount=300, bloomtype=ROW, size=19.1 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731552206756 2024-11-14T02:43:26,958 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bf087ea8c0e4a9cb5d72c60a800b056, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731552206778 2024-11-14T02:43:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T02:43:26,974 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:26,975 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:43:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-14T02:43:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T02:43:26,979 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:43:26,981 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:43:26,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:43:26,994 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#A#compaction#54 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:26,995 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#B#compaction#55 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:26,995 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/86add8e4c0d44806a637a4bd3871462b is 50, key is test_row_0/A:col10/1731552206803/Put/seqid=0 2024-11-14T02:43:26,995 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/955920991cec41999e5478445562b2c7 is 50, key is test_row_0/B:col10/1731552206803/Put/seqid=0 2024-11-14T02:43:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:27,009 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:43:27,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-14T02:43:27,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741892_1068 (size=12865) 2024-11-14T02:43:27,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741891_1067 (size=12865) 2024-11-14T02:43:27,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/17c80e055fd142d0bd9592a94efa410e is 50, key is test_row_0/A:col10/1731552206885/Put/seqid=0 2024-11-14T02:43:27,035 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/955920991cec41999e5478445562b2c7 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/955920991cec41999e5478445562b2c7 2024-11-14T02:43:27,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552267028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:27,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:27,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552267035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:27,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:27,038 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/86add8e4c0d44806a637a4bd3871462b as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/86add8e4c0d44806a637a4bd3871462b 2024-11-14T02:43:27,047 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/B of 557913855dcf97d8b784fddf23bec5ee into 955920991cec41999e5478445562b2c7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:27,047 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:27,047 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/B, priority=12, startTime=1731552206949; duration=0sec 2024-11-14T02:43:27,047 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:27,048 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:B 2024-11-14T02:43:27,048 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:43:27,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552267037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:27,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:27,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552267037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:27,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:27,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552267038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:27,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:27,052 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/A of 557913855dcf97d8b784fddf23bec5ee into 86add8e4c0d44806a637a4bd3871462b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:27,052 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:27,052 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/A, priority=12, startTime=1731552206949; duration=0sec 2024-11-14T02:43:27,052 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:27,052 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:A 2024-11-14T02:43:27,055 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49182 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:43:27,055 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/C is initiating minor compaction (all files) 2024-11-14T02:43:27,055 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/C in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:27,056 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/a6bde6ec23e34293bcb7e9d980319977, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/50b8cc0ae9d44dc9a8511846ffe4c8a7, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/529d6cd9e5e14bf4a256a0b92d20a713, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/26d87dcf48a548699e162a29efe0c994] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=48.0 K 2024-11-14T02:43:27,057 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting a6bde6ec23e34293bcb7e9d980319977, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731552204604 2024-11-14T02:43:27,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741893_1069 (size=12301) 2024-11-14T02:43:27,058 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 50b8cc0ae9d44dc9a8511846ffe4c8a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731552205775 2024-11-14T02:43:27,058 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 529d6cd9e5e14bf4a256a0b92d20a713, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731552206434 2024-11-14T02:43:27,059 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 26d87dcf48a548699e162a29efe0c994, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731552206778 2024-11-14T02:43:27,076 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#C#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:27,077 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/6ec9ccce0f1141019b040af0eff4abc8 is 50, key is test_row_0/C:col10/1731552206803/Put/seqid=0 2024-11-14T02:43:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T02:43:27,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741894_1070 (size=12865) 2024-11-14T02:43:27,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T02:43:27,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:27,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:27,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:27,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:27,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:27,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552267143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552267143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:27,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552267153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552267154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552267154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:27,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T02:43:27,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:27,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:27,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:27,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:27,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:27,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T02:43:27,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:27,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552267353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552267353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:27,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:27,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:27,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:27,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552267364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:27,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:27,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:27,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552267364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:27,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:27,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:27,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552267364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:27,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:27,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T02:43:27,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:27,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:27,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:27,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:27,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:27,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/17c80e055fd142d0bd9592a94efa410e 2024-11-14T02:43:27,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/8a83072f5ebf471098bf4e75566c4b46 is 50, key is test_row_0/B:col10/1731552206885/Put/seqid=0 2024-11-14T02:43:27,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741895_1071 (size=12301) 2024-11-14T02:43:27,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/8a83072f5ebf471098bf4e75566c4b46 2024-11-14T02:43:27,494 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/6ec9ccce0f1141019b040af0eff4abc8 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/6ec9ccce0f1141019b040af0eff4abc8 2024-11-14T02:43:27,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/8d4401cfb16141b19f287dd4e64cc441 is 50, key is test_row_0/C:col10/1731552206885/Put/seqid=0 2024-11-14T02:43:27,505 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/C of 557913855dcf97d8b784fddf23bec5ee into 6ec9ccce0f1141019b040af0eff4abc8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:27,505 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:27,505 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/C, priority=12, startTime=1731552206950; duration=0sec 2024-11-14T02:43:27,505 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:27,505 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:C 2024-11-14T02:43:27,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741896_1072 (size=12301) 2024-11-14T02:43:27,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T02:43:27,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:27,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:27,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:27,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:27,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:27,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T02:43:27,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:27,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552267664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552267664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:27,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:27,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552267673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:27,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:27,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552267673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:27,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:27,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:27,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552267673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:27,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:27,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:27,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:27,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T02:43:27,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:27,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:27,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:27,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:27,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:27,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T02:43:27,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:27,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:27,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:27,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:27,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:27,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/8d4401cfb16141b19f287dd4e64cc441 2024-11-14T02:43:27,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/17c80e055fd142d0bd9592a94efa410e as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/17c80e055fd142d0bd9592a94efa410e 2024-11-14T02:43:27,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/17c80e055fd142d0bd9592a94efa410e, entries=150, sequenceid=289, filesize=12.0 K 2024-11-14T02:43:27,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/8a83072f5ebf471098bf4e75566c4b46 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/8a83072f5ebf471098bf4e75566c4b46 2024-11-14T02:43:27,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/8a83072f5ebf471098bf4e75566c4b46, entries=150, sequenceid=289, filesize=12.0 K 2024-11-14T02:43:27,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/8d4401cfb16141b19f287dd4e64cc441 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/8d4401cfb16141b19f287dd4e64cc441 2024-11-14T02:43:27,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/8d4401cfb16141b19f287dd4e64cc441, entries=150, sequenceid=289, filesize=12.0 K 2024-11-14T02:43:27,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 557913855dcf97d8b784fddf23bec5ee in 934ms, sequenceid=289, compaction requested=false 2024-11-14T02:43:27,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:28,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T02:43:28,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:28,058 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T02:43:28,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/dcdab4dd24cb4080a650155021057cb8 is 50, key is test_row_0/A:col10/1731552207033/Put/seqid=0 2024-11-14T02:43:28,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741897_1073 (size=12301) 2024-11-14T02:43:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T02:43:28,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:28,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:28,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:28,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552268244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:28,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:28,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:28,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552268246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552268248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:28,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552268248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:28,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:28,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552268250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:28,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:28,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552268353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:28,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:28,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552268364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:28,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:28,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552268364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:28,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:28,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552268365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552268365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:28,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:28,471 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/dcdab4dd24cb4080a650155021057cb8 2024-11-14T02:43:28,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/e3e24c9efdd949f28505064c068d735d is 50, key is test_row_0/B:col10/1731552207033/Put/seqid=0 2024-11-14T02:43:28,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741898_1074 (size=12301) 2024-11-14T02:43:28,488 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/e3e24c9efdd949f28505064c068d735d 2024-11-14T02:43:28,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/70b7074f48cd4cb3884458ddc0a486cb is 50, key is test_row_0/C:col10/1731552207033/Put/seqid=0 2024-11-14T02:43:28,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741899_1075 (size=12301) 2024-11-14T02:43:28,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552268563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:28,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:28,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552268583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:28,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:28,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552268584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:28,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:28,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552268584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:28,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:28,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552268584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:28,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:28,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:28,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552268873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:28,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:28,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:28,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552268894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:28,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552268894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:28,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:28,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552268894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552268894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:28,916 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/70b7074f48cd4cb3884458ddc0a486cb 2024-11-14T02:43:28,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/dcdab4dd24cb4080a650155021057cb8 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/dcdab4dd24cb4080a650155021057cb8 2024-11-14T02:43:28,928 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/dcdab4dd24cb4080a650155021057cb8, entries=150, sequenceid=303, filesize=12.0 K 2024-11-14T02:43:28,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/e3e24c9efdd949f28505064c068d735d as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/e3e24c9efdd949f28505064c068d735d 2024-11-14T02:43:28,936 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/e3e24c9efdd949f28505064c068d735d, entries=150, sequenceid=303, filesize=12.0 K 2024-11-14T02:43:28,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/70b7074f48cd4cb3884458ddc0a486cb as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/70b7074f48cd4cb3884458ddc0a486cb 2024-11-14T02:43:28,942 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/70b7074f48cd4cb3884458ddc0a486cb, entries=150, sequenceid=303, filesize=12.0 K 2024-11-14T02:43:28,944 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 557913855dcf97d8b784fddf23bec5ee in 885ms, sequenceid=303, compaction requested=true 2024-11-14T02:43:28,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:28,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:28,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-14T02:43:28,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-14T02:43:28,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-14T02:43:28,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9640 sec 2024-11-14T02:43:28,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.9740 sec 2024-11-14T02:43:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T02:43:29,134 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:29,134 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:43:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-14T02:43:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T02:43:29,136 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:43:29,137 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:43:29,137 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:43:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T02:43:29,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T02:43:29,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:29,291 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T02:43:29,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/9c11f5a7306148b08a1a4d88604fcbad is 50, key is test_row_0/A:col10/1731552208248/Put/seqid=0 2024-11-14T02:43:29,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741900_1076 (size=12301) 2024-11-14T02:43:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:29,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:29,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552269404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:29,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:29,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:29,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:29,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552269404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:29,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:29,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:29,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:29,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552269409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:29,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552269410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:29,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552269411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:29,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:29,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T02:43:29,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552269523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:29,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:29,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:29,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552269523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:29,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:29,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:29,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:29,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:29,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552269523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:29,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:29,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:29,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:29,706 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/9c11f5a7306148b08a1a4d88604fcbad 2024-11-14T02:43:29,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/f82b76aa2f2d4d8582280afaa254daf5 is 50, key is test_row_0/B:col10/1731552208248/Put/seqid=0 2024-11-14T02:43:29,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741901_1077 (size=12301) 2024-11-14T02:43:29,731 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/f82b76aa2f2d4d8582280afaa254daf5 2024-11-14T02:43:29,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:29,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552269733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:29,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:29,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:29,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:29,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:29,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552269733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:29,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:29,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:29,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:29,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552269734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:29,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:29,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:29,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:29,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/cb1cf2b3099843d8a9a25b98df83cd71 is 50, key is test_row_0/C:col10/1731552208248/Put/seqid=0 2024-11-14T02:43:29,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741902_1078 (size=12301) 2024-11-14T02:43:29,761 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/cb1cf2b3099843d8a9a25b98df83cd71 2024-11-14T02:43:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T02:43:29,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/9c11f5a7306148b08a1a4d88604fcbad as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/9c11f5a7306148b08a1a4d88604fcbad 2024-11-14T02:43:29,776 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/9c11f5a7306148b08a1a4d88604fcbad, entries=150, sequenceid=328, filesize=12.0 K 2024-11-14T02:43:29,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/f82b76aa2f2d4d8582280afaa254daf5 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/f82b76aa2f2d4d8582280afaa254daf5 2024-11-14T02:43:29,785 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/f82b76aa2f2d4d8582280afaa254daf5, entries=150, sequenceid=328, filesize=12.0 K 2024-11-14T02:43:29,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/cb1cf2b3099843d8a9a25b98df83cd71 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/cb1cf2b3099843d8a9a25b98df83cd71 2024-11-14T02:43:29,792 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/cb1cf2b3099843d8a9a25b98df83cd71, entries=150, sequenceid=328, filesize=12.0 K 2024-11-14T02:43:29,793 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 557913855dcf97d8b784fddf23bec5ee in 502ms, sequenceid=328, compaction requested=true 2024-11-14T02:43:29,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:29,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:29,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-14T02:43:29,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-14T02:43:29,798 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-14T02:43:29,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 659 msec 2024-11-14T02:43:29,801 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 666 msec 2024-11-14T02:43:30,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:30,046 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:43:30,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T02:43:30,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/8ed908a8a4624e3bac8ad6f1b320eaab is 50, key is test_row_0/A:col10/1731552209400/Put/seqid=0 2024-11-14T02:43:30,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741903_1079 (size=17181) 2024-11-14T02:43:30,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/8ed908a8a4624e3bac8ad6f1b320eaab 2024-11-14T02:43:30,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/9af732d1f3814417af6c1cde786a0e7a is 50, key is test_row_0/B:col10/1731552209400/Put/seqid=0 2024-11-14T02:43:30,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741904_1080 (size=12301) 2024-11-14T02:43:30,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/9af732d1f3814417af6c1cde786a0e7a 2024-11-14T02:43:30,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:30,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552270109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:30,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:30,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:30,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552270111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:30,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:30,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552270111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:30,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:30,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:30,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/558df922b8d741b59817dc51c1b6ded1 is 50, key is test_row_0/C:col10/1731552209400/Put/seqid=0 2024-11-14T02:43:30,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741905_1081 (size=12301) 2024-11-14T02:43:30,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/558df922b8d741b59817dc51c1b6ded1 2024-11-14T02:43:30,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/8ed908a8a4624e3bac8ad6f1b320eaab as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/8ed908a8a4624e3bac8ad6f1b320eaab 2024-11-14T02:43:30,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/8ed908a8a4624e3bac8ad6f1b320eaab, entries=250, sequenceid=341, filesize=16.8 K 2024-11-14T02:43:30,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/9af732d1f3814417af6c1cde786a0e7a as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/9af732d1f3814417af6c1cde786a0e7a 2024-11-14T02:43:30,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/9af732d1f3814417af6c1cde786a0e7a, entries=150, sequenceid=341, filesize=12.0 K 2024-11-14T02:43:30,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/558df922b8d741b59817dc51c1b6ded1 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/558df922b8d741b59817dc51c1b6ded1 2024-11-14T02:43:30,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/558df922b8d741b59817dc51c1b6ded1, entries=150, sequenceid=341, filesize=12.0 K 2024-11-14T02:43:30,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 557913855dcf97d8b784fddf23bec5ee in 140ms, sequenceid=341, compaction requested=true 2024-11-14T02:43:30,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:30,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:43:30,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:30,186 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-14T02:43:30,186 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-14T02:43:30,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:43:30,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:30,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:43:30,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:30,188 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62069 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-14T02:43:30,189 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/B is initiating minor compaction (all files) 2024-11-14T02:43:30,189 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/B in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:30,189 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/955920991cec41999e5478445562b2c7, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/8a83072f5ebf471098bf4e75566c4b46, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/e3e24c9efdd949f28505064c068d735d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/f82b76aa2f2d4d8582280afaa254daf5, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/9af732d1f3814417af6c1cde786a0e7a] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=60.6 K 2024-11-14T02:43:30,190 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 955920991cec41999e5478445562b2c7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731552206778 2024-11-14T02:43:30,191 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 66949 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-14T02:43:30,191 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a83072f5ebf471098bf4e75566c4b46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731552206885 2024-11-14T02:43:30,191 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/A is initiating minor compaction (all files) 2024-11-14T02:43:30,191 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/A in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:30,191 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/86add8e4c0d44806a637a4bd3871462b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/17c80e055fd142d0bd9592a94efa410e, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/dcdab4dd24cb4080a650155021057cb8, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/9c11f5a7306148b08a1a4d88604fcbad, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/8ed908a8a4624e3bac8ad6f1b320eaab] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=65.4 K 2024-11-14T02:43:30,191 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting e3e24c9efdd949f28505064c068d735d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731552207030 2024-11-14T02:43:30,192 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting f82b76aa2f2d4d8582280afaa254daf5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731552208244 2024-11-14T02:43:30,192 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86add8e4c0d44806a637a4bd3871462b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731552206778 2024-11-14T02:43:30,192 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 9af732d1f3814417af6c1cde786a0e7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731552209400 2024-11-14T02:43:30,192 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17c80e055fd142d0bd9592a94efa410e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731552206885 2024-11-14T02:43:30,193 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcdab4dd24cb4080a650155021057cb8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731552207030 2024-11-14T02:43:30,193 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c11f5a7306148b08a1a4d88604fcbad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731552208244 2024-11-14T02:43:30,194 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ed908a8a4624e3bac8ad6f1b320eaab, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731552209400 2024-11-14T02:43:30,235 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#B#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:30,235 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#A#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:30,235 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/1a2330dab0674f88a38eb57f83f6f0c7 is 50, key is test_row_0/B:col10/1731552209400/Put/seqid=0 2024-11-14T02:43:30,236 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/f9396a98d90f443c8dc425ca5a486485 is 50, key is test_row_0/A:col10/1731552209400/Put/seqid=0 2024-11-14T02:43:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:30,260 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:43:30,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-14T02:43:30,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741906_1082 (size=13085) 2024-11-14T02:43:30,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741907_1083 (size=13085) 2024-11-14T02:43:30,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/fcce8d5cbe03454aafcba999f46582ac is 50, key is test_row_0/A:col10/1731552210089/Put/seqid=0 2024-11-14T02:43:30,273 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/f9396a98d90f443c8dc425ca5a486485 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f9396a98d90f443c8dc425ca5a486485 2024-11-14T02:43:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T02:43:30,274 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:30,275 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:43:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-14T02:43:30,278 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:43:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T02:43:30,279 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:43:30,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:43:30,282 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/1a2330dab0674f88a38eb57f83f6f0c7 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/1a2330dab0674f88a38eb57f83f6f0c7 2024-11-14T02:43:30,287 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/A of 557913855dcf97d8b784fddf23bec5ee into f9396a98d90f443c8dc425ca5a486485(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:30,290 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:30,290 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/A, priority=11, startTime=1731552210186; duration=0sec 2024-11-14T02:43:30,290 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:30,290 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:A 2024-11-14T02:43:30,291 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-14T02:43:30,293 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/B of 557913855dcf97d8b784fddf23bec5ee into 1a2330dab0674f88a38eb57f83f6f0c7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:30,293 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:30,293 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/B, priority=11, startTime=1731552210186; duration=0sec 2024-11-14T02:43:30,293 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:30,294 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:B 2024-11-14T02:43:30,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741908_1084 (size=12301) 2024-11-14T02:43:30,296 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62069 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-14T02:43:30,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:30,296 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/C is initiating minor compaction (all files) 2024-11-14T02:43:30,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552270289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:30,296 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/C in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:30,297 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/6ec9ccce0f1141019b040af0eff4abc8, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/8d4401cfb16141b19f287dd4e64cc441, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/70b7074f48cd4cb3884458ddc0a486cb, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/cb1cf2b3099843d8a9a25b98df83cd71, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/558df922b8d741b59817dc51c1b6ded1] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=60.6 K 2024-11-14T02:43:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:30,298 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ec9ccce0f1141019b040af0eff4abc8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731552206778 2024-11-14T02:43:30,298 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d4401cfb16141b19f287dd4e64cc441, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731552206885 2024-11-14T02:43:30,298 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70b7074f48cd4cb3884458ddc0a486cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731552207030 2024-11-14T02:43:30,299 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb1cf2b3099843d8a9a25b98df83cd71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731552208244 2024-11-14T02:43:30,299 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 558df922b8d741b59817dc51c1b6ded1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731552209400 2024-11-14T02:43:30,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:30,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552270293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:30,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:30,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:30,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552270296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:30,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:30,318 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#C#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:30,319 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/a7578354fa5b418a8800cb9a7ec36d59 is 50, key is test_row_0/C:col10/1731552209400/Put/seqid=0 2024-11-14T02:43:30,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741909_1085 (size=13085) 2024-11-14T02:43:30,350 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/a7578354fa5b418a8800cb9a7ec36d59 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/a7578354fa5b418a8800cb9a7ec36d59 2024-11-14T02:43:30,359 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/C of 557913855dcf97d8b784fddf23bec5ee into a7578354fa5b418a8800cb9a7ec36d59(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:30,359 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:30,359 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/C, priority=11, startTime=1731552210187; duration=0sec 2024-11-14T02:43:30,359 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:30,359 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:C 2024-11-14T02:43:30,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T02:43:30,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:30,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552270404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:30,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:30,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:30,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552270425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:30,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:30,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552270425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:30,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:30,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:30,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552270425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:30,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:30,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552270425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:30,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:30,431 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_2' in 557913855dcf97d8b784fddf23bec5ee of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:43:30,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:30,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T02:43:30,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:30,430 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_2' in 557913855dcf97d8b784fddf23bec5ee of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:43:30,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:30,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:30,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:30,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:30,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:30,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:30,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T02:43:30,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:30,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:30,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:30,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:30,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:30,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T02:43:30,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:30,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552270614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:30,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:30,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552270633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552270633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:30,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/fcce8d5cbe03454aafcba999f46582ac 2024-11-14T02:43:30,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/4025094c3ba64143bdd7b80af02116bf is 50, key is test_row_0/B:col10/1731552210089/Put/seqid=0 2024-11-14T02:43:30,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741910_1086 (size=12301) 2024-11-14T02:43:30,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T02:43:30,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:30,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:30,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:30,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:30,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:30,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:30,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T02:43:30,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:30,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:30,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:30,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:30,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:30,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T02:43:30,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552270924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:30,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552270943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:30,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:30,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552270943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:30,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:30,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:30,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:31,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T02:43:31,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:31,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:31,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:31,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:31,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:31,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/4025094c3ba64143bdd7b80af02116bf 2024-11-14T02:43:31,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/df898dc965454ca7953e942967756756 is 50, key is test_row_0/C:col10/1731552210089/Put/seqid=0 2024-11-14T02:43:31,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741911_1087 (size=12301) 2024-11-14T02:43:31,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T02:43:31,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:31,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:31,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:31,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:31,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:31,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:31,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T02:43:31,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:31,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:31,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:31,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:31,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:31,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T02:43:31,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552271433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:31,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:31,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:31,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:31,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552271454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:31,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552271454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:31,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:31,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:31,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:31,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:31,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:31,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:31,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T02:43:31,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:31,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:31,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:31,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:31,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:31,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/df898dc965454ca7953e942967756756 2024-11-14T02:43:31,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/fcce8d5cbe03454aafcba999f46582ac as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/fcce8d5cbe03454aafcba999f46582ac 2024-11-14T02:43:31,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/fcce8d5cbe03454aafcba999f46582ac, entries=150, sequenceid=368, filesize=12.0 K 2024-11-14T02:43:31,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/4025094c3ba64143bdd7b80af02116bf as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/4025094c3ba64143bdd7b80af02116bf 2024-11-14T02:43:31,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/4025094c3ba64143bdd7b80af02116bf, entries=150, sequenceid=368, filesize=12.0 K 2024-11-14T02:43:31,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/df898dc965454ca7953e942967756756 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/df898dc965454ca7953e942967756756 2024-11-14T02:43:31,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/df898dc965454ca7953e942967756756, entries=150, sequenceid=368, filesize=12.0 K 2024-11-14T02:43:31,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 557913855dcf97d8b784fddf23bec5ee in 1310ms, sequenceid=368, compaction requested=false 2024-11-14T02:43:31,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:31,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T02:43:31,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:31,664 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-14T02:43:31,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/d1cca7797b074c18bb56590418a44fcb is 50, key is test_row_0/A:col10/1731552210275/Put/seqid=0 2024-11-14T02:43:31,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741912_1088 (size=9857) 2024-11-14T02:43:32,102 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/d1cca7797b074c18bb56590418a44fcb 2024-11-14T02:43:32,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/0fbce780c7124dd8ae549bb73340f742 is 50, key is test_row_0/B:col10/1731552210275/Put/seqid=0 2024-11-14T02:43:32,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741913_1089 (size=9857) 2024-11-14T02:43:32,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T02:43:32,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:32,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:32,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:32,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552272480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:32,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:32,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552272483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:32,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552272482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:32,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552272485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:32,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:32,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:32,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:32,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:32,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552272486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:32,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:32,533 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/0fbce780c7124dd8ae549bb73340f742 2024-11-14T02:43:32,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/6a87047a7900477b8d2b7a3c92057e54 is 50, key is test_row_0/C:col10/1731552210275/Put/seqid=0 2024-11-14T02:43:32,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741914_1090 (size=9857) 2024-11-14T02:43:32,551 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/6a87047a7900477b8d2b7a3c92057e54 2024-11-14T02:43:32,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/d1cca7797b074c18bb56590418a44fcb as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/d1cca7797b074c18bb56590418a44fcb 2024-11-14T02:43:32,565 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/d1cca7797b074c18bb56590418a44fcb, entries=100, sequenceid=380, filesize=9.6 K 2024-11-14T02:43:32,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/0fbce780c7124dd8ae549bb73340f742 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/0fbce780c7124dd8ae549bb73340f742 2024-11-14T02:43:32,572 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/0fbce780c7124dd8ae549bb73340f742, entries=100, sequenceid=380, filesize=9.6 K 2024-11-14T02:43:32,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/6a87047a7900477b8d2b7a3c92057e54 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/6a87047a7900477b8d2b7a3c92057e54 2024-11-14T02:43:32,593 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/6a87047a7900477b8d2b7a3c92057e54, entries=100, sequenceid=380, filesize=9.6 K 2024-11-14T02:43:32,595 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 557913855dcf97d8b784fddf23bec5ee in 931ms, sequenceid=380, compaction requested=true 2024-11-14T02:43:32,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:32,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:32,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-14T02:43:32,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-14T02:43:32,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:32,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-14T02:43:32,599 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:43:32,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3180 sec 2024-11-14T02:43:32,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-14T02:43:32,603 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.3270 sec 2024-11-14T02:43:32,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/f8336e82fca748d9b3c3afa8c47fce0e is 50, key is test_row_0/A:col10/1731552212597/Put/seqid=0 2024-11-14T02:43:32,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:32,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552272604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:32,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552272604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:32,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741915_1091 (size=12301) 2024-11-14T02:43:32,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:32,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:32,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552272605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:32,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:32,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/f8336e82fca748d9b3c3afa8c47fce0e 2024-11-14T02:43:32,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:32,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552272610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:32,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:32,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552272611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:32,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:32,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/96b6dec37d76418989d973c0717dc5bb is 50, key is test_row_0/B:col10/1731552212597/Put/seqid=0 2024-11-14T02:43:32,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741916_1092 (size=12301) 2024-11-14T02:43:32,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/96b6dec37d76418989d973c0717dc5bb 2024-11-14T02:43:32,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/03e652d0cedf4468afc0ba0715d9def8 is 50, key is test_row_0/C:col10/1731552212597/Put/seqid=0 2024-11-14T02:43:32,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741917_1093 (size=12301) 2024-11-14T02:43:32,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552272723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:32,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:32,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552272724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:32,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:32,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552272724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:32,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:32,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552272724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:32,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:32,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552272724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:32,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:32,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552272933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552272933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:32,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552272933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:32,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552272933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552272933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/03e652d0cedf4468afc0ba0715d9def8 2024-11-14T02:43:33,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/f8336e82fca748d9b3c3afa8c47fce0e as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f8336e82fca748d9b3c3afa8c47fce0e 2024-11-14T02:43:33,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f8336e82fca748d9b3c3afa8c47fce0e, entries=150, sequenceid=408, filesize=12.0 K 2024-11-14T02:43:33,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/96b6dec37d76418989d973c0717dc5bb as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/96b6dec37d76418989d973c0717dc5bb 2024-11-14T02:43:33,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/96b6dec37d76418989d973c0717dc5bb, entries=150, sequenceid=408, filesize=12.0 K 2024-11-14T02:43:33,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/03e652d0cedf4468afc0ba0715d9def8 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/03e652d0cedf4468afc0ba0715d9def8 2024-11-14T02:43:33,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/03e652d0cedf4468afc0ba0715d9def8, entries=150, sequenceid=408, filesize=12.0 K 2024-11-14T02:43:33,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 557913855dcf97d8b784fddf23bec5ee in 473ms, sequenceid=408, compaction requested=true 2024-11-14T02:43:33,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:33,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:43:33,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:33,072 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:43:33,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:43:33,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:33,072 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:43:33,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:43:33,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:33,074 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:43:33,074 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/A is initiating minor compaction (all files) 2024-11-14T02:43:33,075 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/A in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:33,075 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f9396a98d90f443c8dc425ca5a486485, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/fcce8d5cbe03454aafcba999f46582ac, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/d1cca7797b074c18bb56590418a44fcb, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f8336e82fca748d9b3c3afa8c47fce0e] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=46.4 K 2024-11-14T02:43:33,076 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9396a98d90f443c8dc425ca5a486485, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731552209400 2024-11-14T02:43:33,076 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:43:33,076 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/B is initiating minor compaction (all files) 2024-11-14T02:43:33,076 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/B in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:33,076 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/1a2330dab0674f88a38eb57f83f6f0c7, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/4025094c3ba64143bdd7b80af02116bf, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/0fbce780c7124dd8ae549bb73340f742, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/96b6dec37d76418989d973c0717dc5bb] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=46.4 K 2024-11-14T02:43:33,076 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcce8d5cbe03454aafcba999f46582ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731552210089 2024-11-14T02:43:33,077 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a2330dab0674f88a38eb57f83f6f0c7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731552209400 2024-11-14T02:43:33,077 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1cca7797b074c18bb56590418a44fcb, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731552210275 2024-11-14T02:43:33,077 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 4025094c3ba64143bdd7b80af02116bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731552210089 2024-11-14T02:43:33,077 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8336e82fca748d9b3c3afa8c47fce0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731552212478 2024-11-14T02:43:33,077 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fbce780c7124dd8ae549bb73340f742, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731552210275 2024-11-14T02:43:33,078 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 96b6dec37d76418989d973c0717dc5bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731552212478 2024-11-14T02:43:33,092 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#A#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:33,093 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/925f6c3565474b089088b61f968cf2b6 is 50, key is test_row_0/A:col10/1731552212597/Put/seqid=0 2024-11-14T02:43:33,097 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#B#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:33,098 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/461ecbdaf1d04b82bb11200a8f1d32b5 is 50, key is test_row_0/B:col10/1731552212597/Put/seqid=0 2024-11-14T02:43:33,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741918_1094 (size=13221) 2024-11-14T02:43:33,128 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/925f6c3565474b089088b61f968cf2b6 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/925f6c3565474b089088b61f968cf2b6 2024-11-14T02:43:33,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741919_1095 (size=13221) 2024-11-14T02:43:33,136 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/A of 557913855dcf97d8b784fddf23bec5ee into 925f6c3565474b089088b61f968cf2b6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:33,137 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:33,137 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/A, priority=12, startTime=1731552213072; duration=0sec 2024-11-14T02:43:33,139 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:33,139 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:A 2024-11-14T02:43:33,139 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:43:33,142 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:43:33,142 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/C is initiating minor compaction (all files) 2024-11-14T02:43:33,142 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/C in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:33,142 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/a7578354fa5b418a8800cb9a7ec36d59, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/df898dc965454ca7953e942967756756, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/6a87047a7900477b8d2b7a3c92057e54, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/03e652d0cedf4468afc0ba0715d9def8] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=46.4 K 2024-11-14T02:43:33,143 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7578354fa5b418a8800cb9a7ec36d59, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731552209400 2024-11-14T02:43:33,145 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting df898dc965454ca7953e942967756756, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731552210089 2024-11-14T02:43:33,145 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/461ecbdaf1d04b82bb11200a8f1d32b5 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/461ecbdaf1d04b82bb11200a8f1d32b5 2024-11-14T02:43:33,145 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a87047a7900477b8d2b7a3c92057e54, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731552210275 2024-11-14T02:43:33,146 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03e652d0cedf4468afc0ba0715d9def8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731552212478 2024-11-14T02:43:33,157 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/B of 557913855dcf97d8b784fddf23bec5ee into 461ecbdaf1d04b82bb11200a8f1d32b5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:33,157 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:33,157 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/B, priority=12, startTime=1731552213072; duration=0sec 2024-11-14T02:43:33,157 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:33,157 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:B 2024-11-14T02:43:33,167 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#C#compaction#83 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:33,167 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/3428be9bef524115821730e583351a2d is 50, key is test_row_0/C:col10/1731552212597/Put/seqid=0 2024-11-14T02:43:33,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741920_1096 (size=13221) 2024-11-14T02:43:33,187 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/3428be9bef524115821730e583351a2d as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/3428be9bef524115821730e583351a2d 2024-11-14T02:43:33,194 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/C of 557913855dcf97d8b784fddf23bec5ee into 3428be9bef524115821730e583351a2d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:33,194 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:33,194 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/C, priority=12, startTime=1731552213072; duration=0sec 2024-11-14T02:43:33,194 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:33,194 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:C 2024-11-14T02:43:33,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:33,247 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:43:33,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T02:43:33,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/c822b89667954818b649d1220a077db6 is 50, key is test_row_0/A:col10/1731552213246/Put/seqid=0 2024-11-14T02:43:33,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:33,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552273275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:33,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552273275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:33,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552273281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:33,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:33,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552273281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:33,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:33,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552273282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741921_1097 (size=12301) 2024-11-14T02:43:33,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/c822b89667954818b649d1220a077db6 2024-11-14T02:43:33,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/e0b3bbf4c21d440da1c7f6f70f595b17 is 50, key is test_row_0/B:col10/1731552213246/Put/seqid=0 2024-11-14T02:43:33,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741922_1098 (size=12301) 2024-11-14T02:43:33,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/e0b3bbf4c21d440da1c7f6f70f595b17 2024-11-14T02:43:33,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/c34af51de19f41be8c3884625304e293 is 50, key is test_row_0/C:col10/1731552213246/Put/seqid=0 2024-11-14T02:43:33,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741923_1099 (size=12301) 2024-11-14T02:43:33,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/c34af51de19f41be8c3884625304e293 2024-11-14T02:43:33,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552273393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:33,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:33,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552273393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:33,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:33,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:33,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552273394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:33,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552273393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552273394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/c822b89667954818b649d1220a077db6 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/c822b89667954818b649d1220a077db6 2024-11-14T02:43:33,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/c822b89667954818b649d1220a077db6, entries=150, sequenceid=423, filesize=12.0 K 2024-11-14T02:43:33,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/e0b3bbf4c21d440da1c7f6f70f595b17 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/e0b3bbf4c21d440da1c7f6f70f595b17 2024-11-14T02:43:33,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/e0b3bbf4c21d440da1c7f6f70f595b17, entries=150, sequenceid=423, filesize=12.0 K 2024-11-14T02:43:33,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/c34af51de19f41be8c3884625304e293 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/c34af51de19f41be8c3884625304e293 2024-11-14T02:43:33,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/c34af51de19f41be8c3884625304e293, entries=150, sequenceid=423, filesize=12.0 K 2024-11-14T02:43:33,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 557913855dcf97d8b784fddf23bec5ee in 191ms, sequenceid=423, compaction requested=false 2024-11-14T02:43:33,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:33,607 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:43:33,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T02:43:33,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/a9b51581b7984e5986ea5e855ac5d2bc is 50, key is test_row_0/A:col10/1731552213605/Put/seqid=0 2024-11-14T02:43:33,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:33,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552273611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:33,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741924_1100 (size=12301) 2024-11-14T02:43:33,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:33,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552273617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:33,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:33,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:33,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552273619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:33,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552273617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:33,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:33,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552273620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:33,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:33,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552273723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:33,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:33,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552273734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552273734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:33,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:33,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552273734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:33,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552273734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:33,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552273934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:33,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:33,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552273943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:33,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552273943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:33,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:33,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552273943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552273943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:34,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/a9b51581b7984e5986ea5e855ac5d2bc 2024-11-14T02:43:34,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/33c255351e204f01a1fdeded6ba94f61 is 50, key is test_row_0/B:col10/1731552213605/Put/seqid=0 2024-11-14T02:43:34,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741925_1101 (size=12301) 2024-11-14T02:43:34,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:34,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552274244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:34,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:34,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:34,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552274253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:34,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:34,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:34,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552274253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:34,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:34,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:34,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552274253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:34,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552274253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:34,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T02:43:34,433 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:34,434 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:43:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-14T02:43:34,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T02:43:34,435 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:43:34,436 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:43:34,436 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:43:34,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/33c255351e204f01a1fdeded6ba94f61 2024-11-14T02:43:34,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/83d6569c79ab4d78a23d384b67afa00f is 50, key is test_row_0/C:col10/1731552213605/Put/seqid=0 2024-11-14T02:43:34,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741926_1102 (size=12301) 2024-11-14T02:43:34,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T02:43:34,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T02:43:34,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:34,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:34,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:34,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T02:43:34,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:34,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:34,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:34,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T02:43:34,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:34,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552274753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:34,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:34,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552274763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:34,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:34,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:34,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552274763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:34,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552274763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:34,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552274763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:34,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:34,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:34,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:34,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/83d6569c79ab4d78a23d384b67afa00f 2024-11-14T02:43:34,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/a9b51581b7984e5986ea5e855ac5d2bc as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/a9b51581b7984e5986ea5e855ac5d2bc 2024-11-14T02:43:34,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/a9b51581b7984e5986ea5e855ac5d2bc, entries=150, sequenceid=449, filesize=12.0 K 2024-11-14T02:43:34,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/33c255351e204f01a1fdeded6ba94f61 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/33c255351e204f01a1fdeded6ba94f61 2024-11-14T02:43:34,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/33c255351e204f01a1fdeded6ba94f61, entries=150, sequenceid=449, filesize=12.0 K 2024-11-14T02:43:34,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/83d6569c79ab4d78a23d384b67afa00f as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/83d6569c79ab4d78a23d384b67afa00f 2024-11-14T02:43:34,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T02:43:34,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:34,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:34,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:34,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/83d6569c79ab4d78a23d384b67afa00f, entries=150, sequenceid=449, filesize=12.0 K 2024-11-14T02:43:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 557913855dcf97d8b784fddf23bec5ee in 1293ms, sequenceid=449, compaction requested=true 2024-11-14T02:43:34,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:34,900 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:34,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:43:34,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:34,901 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:34,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:43:34,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:34,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:43:34,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:34,904 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:34,904 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/A is initiating minor compaction (all files) 2024-11-14T02:43:34,904 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/A in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:34,905 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/925f6c3565474b089088b61f968cf2b6, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/c822b89667954818b649d1220a077db6, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/a9b51581b7984e5986ea5e855ac5d2bc] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=36.9 K 2024-11-14T02:43:34,905 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:34,905 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/B is initiating minor compaction (all files) 2024-11-14T02:43:34,905 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/B in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:34,906 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/461ecbdaf1d04b82bb11200a8f1d32b5, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/e0b3bbf4c21d440da1c7f6f70f595b17, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/33c255351e204f01a1fdeded6ba94f61] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=36.9 K 2024-11-14T02:43:34,906 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 925f6c3565474b089088b61f968cf2b6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731552212478 2024-11-14T02:43:34,906 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 461ecbdaf1d04b82bb11200a8f1d32b5, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731552212478 2024-11-14T02:43:34,907 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting c822b89667954818b649d1220a077db6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731552212604 2024-11-14T02:43:34,907 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting e0b3bbf4c21d440da1c7f6f70f595b17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731552212604 2024-11-14T02:43:34,907 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9b51581b7984e5986ea5e855ac5d2bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731552213273 2024-11-14T02:43:34,908 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 33c255351e204f01a1fdeded6ba94f61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731552213273 2024-11-14T02:43:34,922 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#A#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:34,923 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/c97244dc5b6f4855899bcd8542cc1264 is 50, key is test_row_0/A:col10/1731552213605/Put/seqid=0 2024-11-14T02:43:34,923 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#B#compaction#91 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:34,923 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/63cef4d71cd746a08887e119ee276b1d is 50, key is test_row_0/B:col10/1731552213605/Put/seqid=0 2024-11-14T02:43:34,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741927_1103 (size=13323) 2024-11-14T02:43:34,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741928_1104 (size=13323) 2024-11-14T02:43:35,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T02:43:35,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:35,050 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-14T02:43:35,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/f38c337952ce4534883d890691417e2d is 50, key is test_row_0/A:col10/1731552213616/Put/seqid=0 2024-11-14T02:43:35,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741929_1105 (size=12301) 2024-11-14T02:43:35,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T02:43:35,373 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/63cef4d71cd746a08887e119ee276b1d as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/63cef4d71cd746a08887e119ee276b1d 2024-11-14T02:43:35,373 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/c97244dc5b6f4855899bcd8542cc1264 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/c97244dc5b6f4855899bcd8542cc1264 2024-11-14T02:43:35,380 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/B of 557913855dcf97d8b784fddf23bec5ee into 63cef4d71cd746a08887e119ee276b1d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:35,380 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/A of 557913855dcf97d8b784fddf23bec5ee into c97244dc5b6f4855899bcd8542cc1264(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:35,380 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:35,380 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:35,380 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/B, priority=13, startTime=1731552214901; duration=0sec 2024-11-14T02:43:35,380 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/A, priority=13, startTime=1731552214900; duration=0sec 2024-11-14T02:43:35,380 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:35,380 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:B 2024-11-14T02:43:35,380 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:35,380 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:A 2024-11-14T02:43:35,380 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:35,382 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:35,382 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/C is initiating minor compaction (all files) 2024-11-14T02:43:35,382 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/C in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:35,382 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/3428be9bef524115821730e583351a2d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/c34af51de19f41be8c3884625304e293, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/83d6569c79ab4d78a23d384b67afa00f] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=36.9 K 2024-11-14T02:43:35,383 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 3428be9bef524115821730e583351a2d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731552212478 2024-11-14T02:43:35,383 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting c34af51de19f41be8c3884625304e293, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731552212604 2024-11-14T02:43:35,383 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 83d6569c79ab4d78a23d384b67afa00f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731552213273 2024-11-14T02:43:35,391 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#C#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:35,392 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/49a1a6b94fcb4d22bc61a3dc481b9cc7 is 50, key is test_row_0/C:col10/1731552213605/Put/seqid=0 2024-11-14T02:43:35,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741930_1106 (size=13323) 2024-11-14T02:43:35,459 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/f38c337952ce4534883d890691417e2d 2024-11-14T02:43:35,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/e8f770d79be441e4ba2a2f6349391233 is 50, key is test_row_0/B:col10/1731552213616/Put/seqid=0 2024-11-14T02:43:35,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741931_1107 (size=12301) 2024-11-14T02:43:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T02:43:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:35,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:35,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:35,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552275795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552275793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:35,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:35,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552275796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:35,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:35,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:35,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:35,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552275797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:35,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:35,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:35,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552275799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:35,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:35,805 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/49a1a6b94fcb4d22bc61a3dc481b9cc7 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/49a1a6b94fcb4d22bc61a3dc481b9cc7 2024-11-14T02:43:35,812 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/C of 557913855dcf97d8b784fddf23bec5ee into 49a1a6b94fcb4d22bc61a3dc481b9cc7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:35,812 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:35,812 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/C, priority=13, startTime=1731552214902; duration=0sec 2024-11-14T02:43:35,812 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:35,812 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:C 2024-11-14T02:43:35,875 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/e8f770d79be441e4ba2a2f6349391233 2024-11-14T02:43:35,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/bde4f9e2cc25499eb7707395870d3732 is 50, key is test_row_0/C:col10/1731552213616/Put/seqid=0 2024-11-14T02:43:35,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741932_1108 (size=12301) 2024-11-14T02:43:35,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552275904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:35,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:35,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552275905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:35,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552275905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:35,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:35,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552275906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:35,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:35,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:35,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552275913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:35,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552276113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:36,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:36,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552276113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:36,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552276113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:36,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552276113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:36,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552276124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:36,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,294 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/bde4f9e2cc25499eb7707395870d3732 2024-11-14T02:43:36,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/f38c337952ce4534883d890691417e2d as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f38c337952ce4534883d890691417e2d 2024-11-14T02:43:36,305 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f38c337952ce4534883d890691417e2d, entries=150, sequenceid=459, filesize=12.0 K 2024-11-14T02:43:36,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/e8f770d79be441e4ba2a2f6349391233 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/e8f770d79be441e4ba2a2f6349391233 2024-11-14T02:43:36,312 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/e8f770d79be441e4ba2a2f6349391233, entries=150, sequenceid=459, filesize=12.0 K 2024-11-14T02:43:36,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/bde4f9e2cc25499eb7707395870d3732 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/bde4f9e2cc25499eb7707395870d3732 2024-11-14T02:43:36,320 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/bde4f9e2cc25499eb7707395870d3732, entries=150, sequenceid=459, filesize=12.0 K 2024-11-14T02:43:36,321 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 557913855dcf97d8b784fddf23bec5ee in 1271ms, sequenceid=459, compaction requested=false 2024-11-14T02:43:36,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:36,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:36,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-14T02:43:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-14T02:43:36,325 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-14T02:43:36,325 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8870 sec 2024-11-14T02:43:36,326 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.8910 sec 2024-11-14T02:43:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:36,426 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:43:36,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-14T02:43:36,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/ba85b09d77c844ccb3e9bd6a3c991f50 is 50, key is test_row_0/A:col10/1731552216424/Put/seqid=0 2024-11-14T02:43:36,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741933_1109 (size=12301) 2024-11-14T02:43:36,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:36,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552276429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:36,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552276431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:36,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552276433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:36,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552276436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:36,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:36,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552276436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:36,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:36,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552276543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552276544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:36,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552276544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552276544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T02:43:36,583 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:36,584 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:43:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-14T02:43:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T02:43:36,586 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:43:36,586 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:43:36,586 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:43:36,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T02:43:36,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-14T02:43:36,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:36,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:36,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:36,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:36,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:36,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:36,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552276753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:36,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552276754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552276754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552276754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/ba85b09d77c844ccb3e9bd6a3c991f50 2024-11-14T02:43:36,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/ede5f4b7a8814641b4bfbab6044f7cc6 is 50, key is test_row_0/B:col10/1731552216424/Put/seqid=0 2024-11-14T02:43:36,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741934_1110 (size=12301) 2024-11-14T02:43:36,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-14T02:43:36,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:36,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:36,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:36,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:36,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:36,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:36,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T02:43:36,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:36,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552276943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:36,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:37,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-14T02:43:37,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:37,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:37,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:37,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:37,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:37,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:37,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:37,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552277063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:37,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:37,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:37,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552277063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:37,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:37,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:37,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:37,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552277064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:37,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552277064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:37,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:37,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:37,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-14T02:43:37,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:37,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:37,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:37,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:37,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T02:43:37,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/ede5f4b7a8814641b4bfbab6044f7cc6 2024-11-14T02:43:37,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/5e87ccb233cf4ebfb08ddbd3e513e028 is 50, key is test_row_0/C:col10/1731552216424/Put/seqid=0 2024-11-14T02:43:37,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741935_1111 (size=12301) 2024-11-14T02:43:37,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/5e87ccb233cf4ebfb08ddbd3e513e028 2024-11-14T02:43:37,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/ba85b09d77c844ccb3e9bd6a3c991f50 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/ba85b09d77c844ccb3e9bd6a3c991f50 2024-11-14T02:43:37,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/ba85b09d77c844ccb3e9bd6a3c991f50, entries=150, sequenceid=490, filesize=12.0 K 2024-11-14T02:43:37,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/ede5f4b7a8814641b4bfbab6044f7cc6 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/ede5f4b7a8814641b4bfbab6044f7cc6 2024-11-14T02:43:37,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/ede5f4b7a8814641b4bfbab6044f7cc6, entries=150, sequenceid=490, filesize=12.0 K 2024-11-14T02:43:37,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/5e87ccb233cf4ebfb08ddbd3e513e028 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/5e87ccb233cf4ebfb08ddbd3e513e028 2024-11-14T02:43:37,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/5e87ccb233cf4ebfb08ddbd3e513e028, entries=150, sequenceid=490, filesize=12.0 K 2024-11-14T02:43:37,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 557913855dcf97d8b784fddf23bec5ee in 875ms, sequenceid=490, compaction requested=true 2024-11-14T02:43:37,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:37,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:43:37,302 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:37,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:37,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:43:37,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:37,302 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:37,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:43:37,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:37,303 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:37,303 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:37,303 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/A is initiating minor compaction (all files) 2024-11-14T02:43:37,303 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/B is initiating minor compaction (all files) 2024-11-14T02:43:37,303 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/A in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:37,303 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/B in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:37,303 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/c97244dc5b6f4855899bcd8542cc1264, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f38c337952ce4534883d890691417e2d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/ba85b09d77c844ccb3e9bd6a3c991f50] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=37.0 K 2024-11-14T02:43:37,303 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/63cef4d71cd746a08887e119ee276b1d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/e8f770d79be441e4ba2a2f6349391233, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/ede5f4b7a8814641b4bfbab6044f7cc6] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=37.0 K 2024-11-14T02:43:37,304 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 63cef4d71cd746a08887e119ee276b1d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731552213273 2024-11-14T02:43:37,304 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting c97244dc5b6f4855899bcd8542cc1264, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731552213273 2024-11-14T02:43:37,304 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting e8f770d79be441e4ba2a2f6349391233, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731552213608 2024-11-14T02:43:37,304 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting f38c337952ce4534883d890691417e2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731552213608 2024-11-14T02:43:37,305 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting ede5f4b7a8814641b4bfbab6044f7cc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1731552215797 2024-11-14T02:43:37,305 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba85b09d77c844ccb3e9bd6a3c991f50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1731552215797 2024-11-14T02:43:37,318 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#B#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:37,318 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/8e38c0ac6e4c4a3890f75f82e856936f is 50, key is test_row_0/B:col10/1731552216424/Put/seqid=0 2024-11-14T02:43:37,319 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#A#compaction#100 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:37,319 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/f633d1d222de434b89ab4f93afbca6ae is 50, key is test_row_0/A:col10/1731552216424/Put/seqid=0 2024-11-14T02:43:37,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741936_1112 (size=13425) 2024-11-14T02:43:37,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741937_1113 (size=13425) 2024-11-14T02:43:37,338 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/8e38c0ac6e4c4a3890f75f82e856936f as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/8e38c0ac6e4c4a3890f75f82e856936f 2024-11-14T02:43:37,340 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/f633d1d222de434b89ab4f93afbca6ae as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f633d1d222de434b89ab4f93afbca6ae 2024-11-14T02:43:37,347 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/B of 557913855dcf97d8b784fddf23bec5ee into 8e38c0ac6e4c4a3890f75f82e856936f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:37,348 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:37,348 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/B, priority=13, startTime=1731552217302; duration=0sec 2024-11-14T02:43:37,348 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:37,348 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:B 2024-11-14T02:43:37,348 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:37,349 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/A of 557913855dcf97d8b784fddf23bec5ee into f633d1d222de434b89ab4f93afbca6ae(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:37,350 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:37,350 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/A, priority=13, startTime=1731552217302; duration=0sec 2024-11-14T02:43:37,350 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:37,350 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:A 2024-11-14T02:43:37,351 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:37,351 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/C is initiating minor compaction (all files) 2024-11-14T02:43:37,351 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/C in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:37,351 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/49a1a6b94fcb4d22bc61a3dc481b9cc7, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/bde4f9e2cc25499eb7707395870d3732, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/5e87ccb233cf4ebfb08ddbd3e513e028] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=37.0 K 2024-11-14T02:43:37,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-14T02:43:37,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:37,352 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 49a1a6b94fcb4d22bc61a3dc481b9cc7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731552213273 2024-11-14T02:43:37,352 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-14T02:43:37,352 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting bde4f9e2cc25499eb7707395870d3732, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731552213608 2024-11-14T02:43:37,353 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e87ccb233cf4ebfb08ddbd3e513e028, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1731552215797 2024-11-14T02:43:37,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/a05e7ac317224272b660f3a825bf11cb is 50, key is test_row_0/A:col10/1731552216426/Put/seqid=0 2024-11-14T02:43:37,369 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#C#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:37,370 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/23f4a52ff08d40da91666de75f7fa8f1 is 50, key is test_row_0/C:col10/1731552216424/Put/seqid=0 2024-11-14T02:43:37,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741938_1114 (size=12301) 2024-11-14T02:43:37,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741939_1115 (size=13425) 2024-11-14T02:43:37,394 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/23f4a52ff08d40da91666de75f7fa8f1 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/23f4a52ff08d40da91666de75f7fa8f1 2024-11-14T02:43:37,400 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/C of 557913855dcf97d8b784fddf23bec5ee into 23f4a52ff08d40da91666de75f7fa8f1(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:37,400 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:37,400 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/C, priority=13, startTime=1731552217302; duration=0sec 2024-11-14T02:43:37,401 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:37,401 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:C 2024-11-14T02:43:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:37,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. as already flushing 2024-11-14T02:43:37,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:37,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552277598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:37,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:37,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552277599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:37,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552277600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:37,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:37,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552277601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:37,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:37,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552277703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:37,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:37,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552277703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:37,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:37,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:37,711 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-14T02:43:37,711 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:43:37,713 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-14T02:43:37,713 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:43:37,713 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-14T02:43:37,713 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:43:37,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:37,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:37,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552277713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:37,714 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-14T02:43:37,714 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:43:37,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552277713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:37,716 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:37,716 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:37,716 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:37,716 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:37,716 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:37,716 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:37,716 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:37,716 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:43:37,716 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:43:37,716 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:43:37,717 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:37,717 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:43:37,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T02:43:37,782 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/a05e7ac317224272b660f3a825bf11cb 2024-11-14T02:43:37,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/df4e6e431edc4214abd0a9ea8a125888 is 50, key is test_row_0/B:col10/1731552216426/Put/seqid=0 2024-11-14T02:43:37,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741940_1116 (size=12301) 2024-11-14T02:43:37,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:37,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:37,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552277914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:37,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552277914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:37,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:37,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:37,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:37,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:37,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552277923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:37,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552277924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:37,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:37,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:37,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1731552277953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:37,955 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_1' in 557913855dcf97d8b784fddf23bec5ee of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:43:37,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:38,200 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/df4e6e431edc4214abd0a9ea8a125888 2024-11-14T02:43:38,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/8f21c0ffec9342a6a46972a8cd4e1556 is 50, key is test_row_0/C:col10/1731552216426/Put/seqid=0 2024-11-14T02:43:38,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741941_1117 (size=12301) 2024-11-14T02:43:38,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:38,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:38,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57594 deadline: 1731552278224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:38,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57618 deadline: 1731552278224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:38,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:38,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:38,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:38,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:38,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731552278233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:38,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57576 deadline: 1731552278233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:38,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:38,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:38,622 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/8f21c0ffec9342a6a46972a8cd4e1556 2024-11-14T02:43:38,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/a05e7ac317224272b660f3a825bf11cb as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/a05e7ac317224272b660f3a825bf11cb 2024-11-14T02:43:38,642 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/a05e7ac317224272b660f3a825bf11cb, entries=150, sequenceid=500, filesize=12.0 K 2024-11-14T02:43:38,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/df4e6e431edc4214abd0a9ea8a125888 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/df4e6e431edc4214abd0a9ea8a125888 2024-11-14T02:43:38,649 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/df4e6e431edc4214abd0a9ea8a125888, entries=150, sequenceid=500, filesize=12.0 K 2024-11-14T02:43:38,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/8f21c0ffec9342a6a46972a8cd4e1556 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/8f21c0ffec9342a6a46972a8cd4e1556 2024-11-14T02:43:38,656 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/8f21c0ffec9342a6a46972a8cd4e1556, entries=150, sequenceid=500, filesize=12.0 K 2024-11-14T02:43:38,658 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 557913855dcf97d8b784fddf23bec5ee in 1305ms, sequenceid=500, compaction requested=false 2024-11-14T02:43:38,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:38,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:38,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-14T02:43:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-14T02:43:38,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-14T02:43:38,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0730 sec 2024-11-14T02:43:38,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.0780 sec 2024-11-14T02:43:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T02:43:38,734 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:38,737 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:43:38,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-14T02:43:38,737 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-14T02:43:38,737 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-14T02:43:38,737 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:43:38,737 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:43:38,737 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:38,737 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:38,737 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:38,737 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:38,737 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:43:38,737 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:43:38,745 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-14T02:43:38,745 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-14T02:43:38,745 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:43:38,745 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:43:38,745 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:38,745 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:38,745 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:38,745 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:38,745 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:43:38,746 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:43:38,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/0e1be4efdd1849b293d50c01c15c77e4 is 50, key is test_row_0/A:col10/1731552218734/Put/seqid=0 2024-11-14T02:43:38,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741942_1118 (size=12301) 2024-11-14T02:43:39,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/0e1be4efdd1849b293d50c01c15c77e4 2024-11-14T02:43:39,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/9bcd0ad6721a4c759d37d1c62b45fd39 is 50, key is test_row_0/B:col10/1731552218734/Put/seqid=0 2024-11-14T02:43:39,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741943_1119 (size=12301) 2024-11-14T02:43:39,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/9bcd0ad6721a4c759d37d1c62b45fd39 2024-11-14T02:43:39,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/2df9a4364717410bb24d0d8ff4c09d1b is 50, key is test_row_0/C:col10/1731552218734/Put/seqid=0 2024-11-14T02:43:39,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741944_1120 (size=12301) 2024-11-14T02:43:39,749 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T02:43:39,977 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-14T02:43:39,977 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:43:39,977 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:39,977 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:39,977 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:43:39,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T02:43:39,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-14T02:43:39,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-14T02:43:39,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-14T02:43:39,979 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-14T02:43:39,979 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-14T02:43:39,979 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T02:43:39,979 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4181 2024-11-14T02:43:39,979 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3851 2024-11-14T02:43:39,979 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T02:43:39,979 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1793 2024-11-14T02:43:39,979 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5378 rows 2024-11-14T02:43:39,980 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1804 2024-11-14T02:43:39,980 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5411 rows 2024-11-14T02:43:39,980 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T02:43:39,980 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T02:43:39,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:39,982 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:39,982 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:43:39,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/2df9a4364717410bb24d0d8ff4c09d1b 2024-11-14T02:43:39,988 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T02:43:39,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/0e1be4efdd1849b293d50c01c15c77e4 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/0e1be4efdd1849b293d50c01c15c77e4 2024-11-14T02:43:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T02:43:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-14T02:43:39,996 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552219996"}]},"ts":"1731552219996"} 2024-11-14T02:43:39,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/0e1be4efdd1849b293d50c01c15c77e4, entries=150, sequenceid=531, filesize=12.0 K 2024-11-14T02:43:39,998 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T02:43:39,998 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T02:43:39,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/9bcd0ad6721a4c759d37d1c62b45fd39 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/9bcd0ad6721a4c759d37d1c62b45fd39 2024-11-14T02:43:40,000 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T02:43:40,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/9bcd0ad6721a4c759d37d1c62b45fd39, entries=150, sequenceid=531, filesize=12.0 K 2024-11-14T02:43:40,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/2df9a4364717410bb24d0d8ff4c09d1b as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/2df9a4364717410bb24d0d8ff4c09d1b 2024-11-14T02:43:40,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=557913855dcf97d8b784fddf23bec5ee, UNASSIGN}] 2024-11-14T02:43:40,006 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=557913855dcf97d8b784fddf23bec5ee, UNASSIGN 2024-11-14T02:43:40,008 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=557913855dcf97d8b784fddf23bec5ee, regionState=CLOSING, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:40,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=557913855dcf97d8b784fddf23bec5ee, UNASSIGN because future has completed 2024-11-14T02:43:40,010 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T02:43:40,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259}] 2024-11-14T02:43:40,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/2df9a4364717410bb24d0d8ff4c09d1b, entries=150, sequenceid=531, filesize=12.0 K 2024-11-14T02:43:40,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 557913855dcf97d8b784fddf23bec5ee in 1275ms, sequenceid=531, compaction requested=true 2024-11-14T02:43:40,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:40,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:43:40,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:40,013 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:40,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:43:40,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:40,013 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:40,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 557913855dcf97d8b784fddf23bec5ee:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:43:40,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:40,014 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:40,014 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:40,014 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/A is initiating minor compaction (all files) 2024-11-14T02:43:40,014 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 557913855dcf97d8b784fddf23bec5ee/B is initiating minor compaction (all files) 2024-11-14T02:43:40,014 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/B in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:40,014 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 557913855dcf97d8b784fddf23bec5ee/A in TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:40,015 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f633d1d222de434b89ab4f93afbca6ae, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/a05e7ac317224272b660f3a825bf11cb, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/0e1be4efdd1849b293d50c01c15c77e4] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=37.1 K 2024-11-14T02:43:40,015 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/8e38c0ac6e4c4a3890f75f82e856936f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/df4e6e431edc4214abd0a9ea8a125888, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/9bcd0ad6721a4c759d37d1c62b45fd39] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp, totalSize=37.1 K 2024-11-14T02:43:40,015 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting f633d1d222de434b89ab4f93afbca6ae, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1731552215797 2024-11-14T02:43:40,015 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e38c0ac6e4c4a3890f75f82e856936f, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1731552215797 2024-11-14T02:43:40,016 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting a05e7ac317224272b660f3a825bf11cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1731552216426 2024-11-14T02:43:40,016 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting df4e6e431edc4214abd0a9ea8a125888, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1731552216426 2024-11-14T02:43:40,016 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e1be4efdd1849b293d50c01c15c77e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1731552217598 2024-11-14T02:43:40,016 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bcd0ad6721a4c759d37d1c62b45fd39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1731552217598 2024-11-14T02:43:40,025 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#B#compaction#108 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:40,025 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/56d6f613ac3748cb82eb9dd918dd3767 is 50, key is test_row_0/B:col10/1731552218734/Put/seqid=0 2024-11-14T02:43:40,025 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 557913855dcf97d8b784fddf23bec5ee#A#compaction#109 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:40,026 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/eb45b2fffcdf425296b9b3d370e77880 is 50, key is test_row_0/A:col10/1731552218734/Put/seqid=0 2024-11-14T02:43:40,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741945_1121 (size=13527) 2024-11-14T02:43:40,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741946_1122 (size=13527) 2024-11-14T02:43:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-14T02:43:40,168 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:40,168 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T02:43:40,169 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 557913855dcf97d8b784fddf23bec5ee, disabling compactions & flushes 2024-11-14T02:43:40,169 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-14T02:43:40,441 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/56d6f613ac3748cb82eb9dd918dd3767 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/56d6f613ac3748cb82eb9dd918dd3767 2024-11-14T02:43:40,442 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/eb45b2fffcdf425296b9b3d370e77880 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/eb45b2fffcdf425296b9b3d370e77880 2024-11-14T02:43:40,448 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/B of 557913855dcf97d8b784fddf23bec5ee into 56d6f613ac3748cb82eb9dd918dd3767(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:40,448 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:40,448 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/B, priority=13, startTime=1731552220013; duration=0sec 2024-11-14T02:43:40,448 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:40,448 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:B 2024-11-14T02:43:40,448 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. because compaction request was cancelled 2024-11-14T02:43:40,448 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:C 2024-11-14T02:43:40,449 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 557913855dcf97d8b784fddf23bec5ee/A of 557913855dcf97d8b784fddf23bec5ee into eb45b2fffcdf425296b9b3d370e77880(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:40,449 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 557913855dcf97d8b784fddf23bec5ee: 2024-11-14T02:43:40,449 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., storeName=557913855dcf97d8b784fddf23bec5ee/A, priority=13, startTime=1731552220012; duration=0sec 2024-11-14T02:43:40,449 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:40,449 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:40,449 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 557913855dcf97d8b784fddf23bec5ee:A 2024-11-14T02:43:40,449 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:40,449 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. after waiting 0 ms 2024-11-14T02:43:40,449 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:40,449 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 557913855dcf97d8b784fddf23bec5ee 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-14T02:43:40,454 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/757e954e5a9641f69df4f851c8517150 is 50, key is test_row_0/A:col10/1731552218743/Put/seqid=0 2024-11-14T02:43:40,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741947_1123 (size=12301) 2024-11-14T02:43:40,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-14T02:43:40,858 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/757e954e5a9641f69df4f851c8517150 2024-11-14T02:43:40,870 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/71fb27a2b09843319ef1a39457db98d0 is 50, key is test_row_0/B:col10/1731552218743/Put/seqid=0 2024-11-14T02:43:40,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741948_1124 (size=12301) 2024-11-14T02:43:41,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-14T02:43:41,278 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/71fb27a2b09843319ef1a39457db98d0 2024-11-14T02:43:41,292 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/4f4dbd6bca8b4538a5c69eb29948eaf3 is 50, key is test_row_0/C:col10/1731552218743/Put/seqid=0 2024-11-14T02:43:41,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741949_1125 (size=12301) 2024-11-14T02:43:41,696 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/4f4dbd6bca8b4538a5c69eb29948eaf3 2024-11-14T02:43:41,707 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/A/757e954e5a9641f69df4f851c8517150 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/757e954e5a9641f69df4f851c8517150 2024-11-14T02:43:41,714 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/757e954e5a9641f69df4f851c8517150, entries=150, sequenceid=539, filesize=12.0 K 2024-11-14T02:43:41,715 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/B/71fb27a2b09843319ef1a39457db98d0 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/71fb27a2b09843319ef1a39457db98d0 2024-11-14T02:43:41,721 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/71fb27a2b09843319ef1a39457db98d0, entries=150, sequenceid=539, filesize=12.0 K 2024-11-14T02:43:41,722 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/.tmp/C/4f4dbd6bca8b4538a5c69eb29948eaf3 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/4f4dbd6bca8b4538a5c69eb29948eaf3 2024-11-14T02:43:41,728 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/4f4dbd6bca8b4538a5c69eb29948eaf3, entries=150, sequenceid=539, filesize=12.0 K 2024-11-14T02:43:41,730 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 557913855dcf97d8b784fddf23bec5ee in 1281ms, sequenceid=539, compaction requested=true 2024-11-14T02:43:41,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/b881d7662bda4652885f29025da03180, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/8c1ea73215ca4128ab404232f6363398, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/a8478e5d937049209a44185cd1a64c0b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f33b7b57c76e4c80b8606a22bd59bfcf, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/d0f5fa4376d4495199278852d38706ea, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/a31fd7a6d1e24bf4879c2e20d87c68a3, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/4a83b0abdc894935bcdf760cf016721d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/689ee3865d3d4c0e8804409c49f607ac, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/ecbb14a46f024f30b5d377b18b765d8d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/e70c6ebda19a46439906e8595c565a21, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/2d99d9085c02429997fe03c428eb901f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/6a1d512972584f64a3aabc6fb84f26d5, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/d95a4005cbe74dc8a5851275c9004eaa, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/8f9f964550bf4f8196361e4c217f1010, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/342e5879b69445678c29ed313706266c, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/85d102cc8b144925ab5073c7cdf6f3e5, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/5844f43649c94ac9a5d28a123021f99a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/5bf45cd31ee348969942efd1246bcfbe, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/86add8e4c0d44806a637a4bd3871462b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/17c80e055fd142d0bd9592a94efa410e, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/dcdab4dd24cb4080a650155021057cb8, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/9c11f5a7306148b08a1a4d88604fcbad, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/8ed908a8a4624e3bac8ad6f1b320eaab, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f9396a98d90f443c8dc425ca5a486485, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/fcce8d5cbe03454aafcba999f46582ac, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/d1cca7797b074c18bb56590418a44fcb, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/925f6c3565474b089088b61f968cf2b6, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f8336e82fca748d9b3c3afa8c47fce0e, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/c822b89667954818b649d1220a077db6, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/c97244dc5b6f4855899bcd8542cc1264, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/a9b51581b7984e5986ea5e855ac5d2bc, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f38c337952ce4534883d890691417e2d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f633d1d222de434b89ab4f93afbca6ae, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/ba85b09d77c844ccb3e9bd6a3c991f50, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/a05e7ac317224272b660f3a825bf11cb, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/0e1be4efdd1849b293d50c01c15c77e4] to archive 2024-11-14T02:43:41,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:43:41,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/b881d7662bda4652885f29025da03180 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/b881d7662bda4652885f29025da03180 2024-11-14T02:43:41,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/8c1ea73215ca4128ab404232f6363398 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/8c1ea73215ca4128ab404232f6363398 2024-11-14T02:43:41,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/a8478e5d937049209a44185cd1a64c0b to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/a8478e5d937049209a44185cd1a64c0b 2024-11-14T02:43:41,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f33b7b57c76e4c80b8606a22bd59bfcf to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f33b7b57c76e4c80b8606a22bd59bfcf 2024-11-14T02:43:41,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/d0f5fa4376d4495199278852d38706ea to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/d0f5fa4376d4495199278852d38706ea 2024-11-14T02:43:41,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/a31fd7a6d1e24bf4879c2e20d87c68a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/a31fd7a6d1e24bf4879c2e20d87c68a3 2024-11-14T02:43:41,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/4a83b0abdc894935bcdf760cf016721d to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/4a83b0abdc894935bcdf760cf016721d 2024-11-14T02:43:41,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/689ee3865d3d4c0e8804409c49f607ac to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/689ee3865d3d4c0e8804409c49f607ac 2024-11-14T02:43:41,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/ecbb14a46f024f30b5d377b18b765d8d to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/ecbb14a46f024f30b5d377b18b765d8d 2024-11-14T02:43:41,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/e70c6ebda19a46439906e8595c565a21 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/e70c6ebda19a46439906e8595c565a21 2024-11-14T02:43:41,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/2d99d9085c02429997fe03c428eb901f to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/2d99d9085c02429997fe03c428eb901f 2024-11-14T02:43:41,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/6a1d512972584f64a3aabc6fb84f26d5 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/6a1d512972584f64a3aabc6fb84f26d5 2024-11-14T02:43:41,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/d95a4005cbe74dc8a5851275c9004eaa to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/d95a4005cbe74dc8a5851275c9004eaa 2024-11-14T02:43:41,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/8f9f964550bf4f8196361e4c217f1010 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/8f9f964550bf4f8196361e4c217f1010 2024-11-14T02:43:41,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/342e5879b69445678c29ed313706266c to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/342e5879b69445678c29ed313706266c 2024-11-14T02:43:41,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/85d102cc8b144925ab5073c7cdf6f3e5 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/85d102cc8b144925ab5073c7cdf6f3e5 2024-11-14T02:43:41,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/5844f43649c94ac9a5d28a123021f99a to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/5844f43649c94ac9a5d28a123021f99a 2024-11-14T02:43:41,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/5bf45cd31ee348969942efd1246bcfbe to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/5bf45cd31ee348969942efd1246bcfbe 2024-11-14T02:43:41,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/86add8e4c0d44806a637a4bd3871462b to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/86add8e4c0d44806a637a4bd3871462b 2024-11-14T02:43:41,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/17c80e055fd142d0bd9592a94efa410e to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/17c80e055fd142d0bd9592a94efa410e 2024-11-14T02:43:41,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/dcdab4dd24cb4080a650155021057cb8 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/dcdab4dd24cb4080a650155021057cb8 2024-11-14T02:43:41,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/9c11f5a7306148b08a1a4d88604fcbad to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/9c11f5a7306148b08a1a4d88604fcbad 2024-11-14T02:43:41,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/8ed908a8a4624e3bac8ad6f1b320eaab to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/8ed908a8a4624e3bac8ad6f1b320eaab 2024-11-14T02:43:41,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f9396a98d90f443c8dc425ca5a486485 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f9396a98d90f443c8dc425ca5a486485 2024-11-14T02:43:41,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/fcce8d5cbe03454aafcba999f46582ac to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/fcce8d5cbe03454aafcba999f46582ac 2024-11-14T02:43:41,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/d1cca7797b074c18bb56590418a44fcb to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/d1cca7797b074c18bb56590418a44fcb 2024-11-14T02:43:41,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/925f6c3565474b089088b61f968cf2b6 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/925f6c3565474b089088b61f968cf2b6 2024-11-14T02:43:41,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f8336e82fca748d9b3c3afa8c47fce0e to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f8336e82fca748d9b3c3afa8c47fce0e 2024-11-14T02:43:41,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/c822b89667954818b649d1220a077db6 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/c822b89667954818b649d1220a077db6 2024-11-14T02:43:41,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/c97244dc5b6f4855899bcd8542cc1264 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/c97244dc5b6f4855899bcd8542cc1264 2024-11-14T02:43:41,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/a9b51581b7984e5986ea5e855ac5d2bc to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/a9b51581b7984e5986ea5e855ac5d2bc 2024-11-14T02:43:41,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f38c337952ce4534883d890691417e2d to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f38c337952ce4534883d890691417e2d 2024-11-14T02:43:41,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f633d1d222de434b89ab4f93afbca6ae to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/f633d1d222de434b89ab4f93afbca6ae 2024-11-14T02:43:41,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/ba85b09d77c844ccb3e9bd6a3c991f50 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/ba85b09d77c844ccb3e9bd6a3c991f50 2024-11-14T02:43:41,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/a05e7ac317224272b660f3a825bf11cb to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/a05e7ac317224272b660f3a825bf11cb 2024-11-14T02:43:41,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/0e1be4efdd1849b293d50c01c15c77e4 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/0e1be4efdd1849b293d50c01c15c77e4 2024-11-14T02:43:41,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/aa2971cb61e64835982161764532ae6f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/de7e0c7e5ffd452f95bef8581834463c, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/38203440bebd42d0b4d7299696c1e5e5, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/6833848658a342dd957b7b509915b3d2, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/b79233acc8f341b682e819dc7eac0de9, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/700e119d2da04e71999e84c0f46194c8, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/5c7662343f5144c3b1a0586e82916bb6, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/d6678907e9094d0da4eb5a460dd881d4, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/9f7968b4061548279020bfe255363e99, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/e943f2bf8f274d68af39a95a8290fb2c, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/11f3d24c6daf48fda91172039dcc9af0, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/5440d96c4de84723bf1b02ecc601e717, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/0ffd665bd244460ea53b05d45a43b81c, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/736054f07d3948e8af6979849d588d5b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/6eb22ed252134022a96196026f655742, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/5c0047a241f64faabd1368754ecbba3b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/402372cf6bfa47a0a724d19ea9a70bd0, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/955920991cec41999e5478445562b2c7, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/7bf087ea8c0e4a9cb5d72c60a800b056, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/8a83072f5ebf471098bf4e75566c4b46, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/e3e24c9efdd949f28505064c068d735d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/f82b76aa2f2d4d8582280afaa254daf5, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/1a2330dab0674f88a38eb57f83f6f0c7, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/9af732d1f3814417af6c1cde786a0e7a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/4025094c3ba64143bdd7b80af02116bf, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/0fbce780c7124dd8ae549bb73340f742, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/461ecbdaf1d04b82bb11200a8f1d32b5, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/96b6dec37d76418989d973c0717dc5bb, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/e0b3bbf4c21d440da1c7f6f70f595b17, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/63cef4d71cd746a08887e119ee276b1d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/33c255351e204f01a1fdeded6ba94f61, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/e8f770d79be441e4ba2a2f6349391233, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/8e38c0ac6e4c4a3890f75f82e856936f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/ede5f4b7a8814641b4bfbab6044f7cc6, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/df4e6e431edc4214abd0a9ea8a125888, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/9bcd0ad6721a4c759d37d1c62b45fd39] to archive 2024-11-14T02:43:41,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:43:41,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/aa2971cb61e64835982161764532ae6f to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/aa2971cb61e64835982161764532ae6f 2024-11-14T02:43:41,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/de7e0c7e5ffd452f95bef8581834463c to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/de7e0c7e5ffd452f95bef8581834463c 2024-11-14T02:43:41,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/38203440bebd42d0b4d7299696c1e5e5 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/38203440bebd42d0b4d7299696c1e5e5 2024-11-14T02:43:41,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/6833848658a342dd957b7b509915b3d2 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/6833848658a342dd957b7b509915b3d2 2024-11-14T02:43:41,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/b79233acc8f341b682e819dc7eac0de9 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/b79233acc8f341b682e819dc7eac0de9 2024-11-14T02:43:41,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/700e119d2da04e71999e84c0f46194c8 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/700e119d2da04e71999e84c0f46194c8 2024-11-14T02:43:41,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/5c7662343f5144c3b1a0586e82916bb6 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/5c7662343f5144c3b1a0586e82916bb6 2024-11-14T02:43:41,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/d6678907e9094d0da4eb5a460dd881d4 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/d6678907e9094d0da4eb5a460dd881d4 2024-11-14T02:43:41,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/9f7968b4061548279020bfe255363e99 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/9f7968b4061548279020bfe255363e99 2024-11-14T02:43:41,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/e943f2bf8f274d68af39a95a8290fb2c to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/e943f2bf8f274d68af39a95a8290fb2c 2024-11-14T02:43:41,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/11f3d24c6daf48fda91172039dcc9af0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/11f3d24c6daf48fda91172039dcc9af0 2024-11-14T02:43:41,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/5440d96c4de84723bf1b02ecc601e717 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/5440d96c4de84723bf1b02ecc601e717 2024-11-14T02:43:41,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/0ffd665bd244460ea53b05d45a43b81c to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/0ffd665bd244460ea53b05d45a43b81c 2024-11-14T02:43:41,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/736054f07d3948e8af6979849d588d5b to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/736054f07d3948e8af6979849d588d5b 2024-11-14T02:43:41,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/6eb22ed252134022a96196026f655742 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/6eb22ed252134022a96196026f655742 2024-11-14T02:43:41,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/5c0047a241f64faabd1368754ecbba3b to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/5c0047a241f64faabd1368754ecbba3b 2024-11-14T02:43:41,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/402372cf6bfa47a0a724d19ea9a70bd0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/402372cf6bfa47a0a724d19ea9a70bd0 2024-11-14T02:43:41,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/955920991cec41999e5478445562b2c7 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/955920991cec41999e5478445562b2c7 2024-11-14T02:43:41,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/7bf087ea8c0e4a9cb5d72c60a800b056 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/7bf087ea8c0e4a9cb5d72c60a800b056 2024-11-14T02:43:41,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/8a83072f5ebf471098bf4e75566c4b46 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/8a83072f5ebf471098bf4e75566c4b46 2024-11-14T02:43:41,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/e3e24c9efdd949f28505064c068d735d to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/e3e24c9efdd949f28505064c068d735d 2024-11-14T02:43:41,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/f82b76aa2f2d4d8582280afaa254daf5 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/f82b76aa2f2d4d8582280afaa254daf5 2024-11-14T02:43:41,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/1a2330dab0674f88a38eb57f83f6f0c7 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/1a2330dab0674f88a38eb57f83f6f0c7 2024-11-14T02:43:41,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/9af732d1f3814417af6c1cde786a0e7a to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/9af732d1f3814417af6c1cde786a0e7a 2024-11-14T02:43:41,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/4025094c3ba64143bdd7b80af02116bf to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/4025094c3ba64143bdd7b80af02116bf 2024-11-14T02:43:41,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/0fbce780c7124dd8ae549bb73340f742 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/0fbce780c7124dd8ae549bb73340f742 2024-11-14T02:43:41,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/461ecbdaf1d04b82bb11200a8f1d32b5 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/461ecbdaf1d04b82bb11200a8f1d32b5 2024-11-14T02:43:41,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/96b6dec37d76418989d973c0717dc5bb to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/96b6dec37d76418989d973c0717dc5bb 2024-11-14T02:43:41,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/e0b3bbf4c21d440da1c7f6f70f595b17 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/e0b3bbf4c21d440da1c7f6f70f595b17 2024-11-14T02:43:41,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/63cef4d71cd746a08887e119ee276b1d to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/63cef4d71cd746a08887e119ee276b1d 2024-11-14T02:43:41,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/33c255351e204f01a1fdeded6ba94f61 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/33c255351e204f01a1fdeded6ba94f61 2024-11-14T02:43:41,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/e8f770d79be441e4ba2a2f6349391233 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/e8f770d79be441e4ba2a2f6349391233 2024-11-14T02:43:41,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/8e38c0ac6e4c4a3890f75f82e856936f to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/8e38c0ac6e4c4a3890f75f82e856936f 2024-11-14T02:43:41,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/ede5f4b7a8814641b4bfbab6044f7cc6 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/ede5f4b7a8814641b4bfbab6044f7cc6 2024-11-14T02:43:41,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/df4e6e431edc4214abd0a9ea8a125888 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/df4e6e431edc4214abd0a9ea8a125888 2024-11-14T02:43:41,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/9bcd0ad6721a4c759d37d1c62b45fd39 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/9bcd0ad6721a4c759d37d1c62b45fd39 2024-11-14T02:43:41,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/ce1771b6fd704a11be22c322100ade66, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/66cd6f8a03ae4aa3a297499a23e9a05a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/c5e84dae5ace43dabae8f90fc58183bf, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/a3147531e1044c0fbb91d087cf03d92b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/b91b0ecbd17948ca944e9840a99f828b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/d31606dfc5d641c3a46489b17d64bc25, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/81e55419b50c4660b7a81f16b9418bdd, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/2a238c9b6d5d425fb593739c700e3128, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/c919e4a563e34727923baa4c0d0ec9e5, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/1f942a02e2e1497390cf3bfe427506a3, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/dd3ffb033cb34e8ca902e6973686608a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/71929cc10d64459eac896ff23ee0d799, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/14bee822d1c5451bba526caf841fd39b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/a6bde6ec23e34293bcb7e9d980319977, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/ddfb02a09b8e4fbaa8da6ab98c65a38c, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/50b8cc0ae9d44dc9a8511846ffe4c8a7, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/529d6cd9e5e14bf4a256a0b92d20a713, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/6ec9ccce0f1141019b040af0eff4abc8, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/26d87dcf48a548699e162a29efe0c994, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/8d4401cfb16141b19f287dd4e64cc441, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/70b7074f48cd4cb3884458ddc0a486cb, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/cb1cf2b3099843d8a9a25b98df83cd71, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/a7578354fa5b418a8800cb9a7ec36d59, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/558df922b8d741b59817dc51c1b6ded1, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/df898dc965454ca7953e942967756756, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/6a87047a7900477b8d2b7a3c92057e54, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/3428be9bef524115821730e583351a2d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/03e652d0cedf4468afc0ba0715d9def8, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/c34af51de19f41be8c3884625304e293, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/49a1a6b94fcb4d22bc61a3dc481b9cc7, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/83d6569c79ab4d78a23d384b67afa00f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/bde4f9e2cc25499eb7707395870d3732, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/5e87ccb233cf4ebfb08ddbd3e513e028] to archive 2024-11-14T02:43:41,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:43:41,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/ce1771b6fd704a11be22c322100ade66 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/ce1771b6fd704a11be22c322100ade66 2024-11-14T02:43:41,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/66cd6f8a03ae4aa3a297499a23e9a05a to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/66cd6f8a03ae4aa3a297499a23e9a05a 2024-11-14T02:43:41,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/c5e84dae5ace43dabae8f90fc58183bf to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/c5e84dae5ace43dabae8f90fc58183bf 2024-11-14T02:43:41,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/a3147531e1044c0fbb91d087cf03d92b to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/a3147531e1044c0fbb91d087cf03d92b 2024-11-14T02:43:41,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/b91b0ecbd17948ca944e9840a99f828b to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/b91b0ecbd17948ca944e9840a99f828b 2024-11-14T02:43:41,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/d31606dfc5d641c3a46489b17d64bc25 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/d31606dfc5d641c3a46489b17d64bc25 2024-11-14T02:43:41,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/81e55419b50c4660b7a81f16b9418bdd to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/81e55419b50c4660b7a81f16b9418bdd 2024-11-14T02:43:41,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/2a238c9b6d5d425fb593739c700e3128 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/2a238c9b6d5d425fb593739c700e3128 2024-11-14T02:43:41,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/c919e4a563e34727923baa4c0d0ec9e5 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/c919e4a563e34727923baa4c0d0ec9e5 2024-11-14T02:43:41,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/1f942a02e2e1497390cf3bfe427506a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/1f942a02e2e1497390cf3bfe427506a3 2024-11-14T02:43:41,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/dd3ffb033cb34e8ca902e6973686608a to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/dd3ffb033cb34e8ca902e6973686608a 2024-11-14T02:43:41,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/71929cc10d64459eac896ff23ee0d799 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/71929cc10d64459eac896ff23ee0d799 2024-11-14T02:43:41,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/14bee822d1c5451bba526caf841fd39b to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/14bee822d1c5451bba526caf841fd39b 2024-11-14T02:43:41,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/a6bde6ec23e34293bcb7e9d980319977 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/a6bde6ec23e34293bcb7e9d980319977 2024-11-14T02:43:41,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/ddfb02a09b8e4fbaa8da6ab98c65a38c to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/ddfb02a09b8e4fbaa8da6ab98c65a38c 2024-11-14T02:43:41,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/50b8cc0ae9d44dc9a8511846ffe4c8a7 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/50b8cc0ae9d44dc9a8511846ffe4c8a7 2024-11-14T02:43:41,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/529d6cd9e5e14bf4a256a0b92d20a713 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/529d6cd9e5e14bf4a256a0b92d20a713 2024-11-14T02:43:41,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/6ec9ccce0f1141019b040af0eff4abc8 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/6ec9ccce0f1141019b040af0eff4abc8 2024-11-14T02:43:41,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/26d87dcf48a548699e162a29efe0c994 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/26d87dcf48a548699e162a29efe0c994 2024-11-14T02:43:41,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/8d4401cfb16141b19f287dd4e64cc441 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/8d4401cfb16141b19f287dd4e64cc441 2024-11-14T02:43:41,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/70b7074f48cd4cb3884458ddc0a486cb to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/70b7074f48cd4cb3884458ddc0a486cb 2024-11-14T02:43:41,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/cb1cf2b3099843d8a9a25b98df83cd71 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/cb1cf2b3099843d8a9a25b98df83cd71 2024-11-14T02:43:41,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/a7578354fa5b418a8800cb9a7ec36d59 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/a7578354fa5b418a8800cb9a7ec36d59 2024-11-14T02:43:41,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/558df922b8d741b59817dc51c1b6ded1 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/558df922b8d741b59817dc51c1b6ded1 2024-11-14T02:43:41,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/df898dc965454ca7953e942967756756 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/df898dc965454ca7953e942967756756 2024-11-14T02:43:41,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/6a87047a7900477b8d2b7a3c92057e54 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/6a87047a7900477b8d2b7a3c92057e54 2024-11-14T02:43:41,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/3428be9bef524115821730e583351a2d to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/3428be9bef524115821730e583351a2d 2024-11-14T02:43:41,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/03e652d0cedf4468afc0ba0715d9def8 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/03e652d0cedf4468afc0ba0715d9def8 2024-11-14T02:43:41,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/c34af51de19f41be8c3884625304e293 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/c34af51de19f41be8c3884625304e293 2024-11-14T02:43:41,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/49a1a6b94fcb4d22bc61a3dc481b9cc7 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/49a1a6b94fcb4d22bc61a3dc481b9cc7 2024-11-14T02:43:41,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/83d6569c79ab4d78a23d384b67afa00f to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/83d6569c79ab4d78a23d384b67afa00f 2024-11-14T02:43:41,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/bde4f9e2cc25499eb7707395870d3732 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/bde4f9e2cc25499eb7707395870d3732 2024-11-14T02:43:41,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/5e87ccb233cf4ebfb08ddbd3e513e028 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/5e87ccb233cf4ebfb08ddbd3e513e028 2024-11-14T02:43:41,883 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/recovered.edits/542.seqid, newMaxSeqId=542, maxSeqId=1 2024-11-14T02:43:41,885 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. 2024-11-14T02:43:41,885 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 557913855dcf97d8b784fddf23bec5ee: Waiting for close lock at 1731552220169Running coprocessor pre-close hooks at 1731552220169Disabling compacts and flushes for region at 1731552220169Disabling writes for close at 1731552220449 (+280 ms)Obtaining lock to block concurrent updates at 1731552220449Preparing flush snapshotting stores in 557913855dcf97d8b784fddf23bec5ee at 1731552220449Finished memstore snapshotting TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731552220450 (+1 ms)Flushing stores of TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee. at 1731552220450Flushing 557913855dcf97d8b784fddf23bec5ee/A: creating writer at 1731552220450Flushing 557913855dcf97d8b784fddf23bec5ee/A: appending metadata at 1731552220453 (+3 ms)Flushing 557913855dcf97d8b784fddf23bec5ee/A: closing flushed file at 1731552220453Flushing 557913855dcf97d8b784fddf23bec5ee/B: creating writer at 1731552220865 (+412 ms)Flushing 557913855dcf97d8b784fddf23bec5ee/B: appending metadata at 1731552220869 (+4 ms)Flushing 557913855dcf97d8b784fddf23bec5ee/B: closing flushed file at 1731552220869Flushing 557913855dcf97d8b784fddf23bec5ee/C: creating writer at 1731552221289 (+420 ms)Flushing 557913855dcf97d8b784fddf23bec5ee/C: appending metadata at 1731552221291 (+2 ms)Flushing 557913855dcf97d8b784fddf23bec5ee/C: closing flushed file at 1731552221291Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18e09c56: reopening flushed file at 1731552221705 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25719863: reopening flushed file at 1731552221714 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8f99a74: reopening flushed file at 1731552221721 (+7 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 557913855dcf97d8b784fddf23bec5ee in 1281ms, sequenceid=539, compaction requested=true at 1731552221730 (+9 ms)Writing region close event to WAL at 1731552221880 (+150 ms)Running coprocessor post-close hooks at 1731552221883 (+3 ms)Closed at 1731552221885 (+2 ms) 2024-11-14T02:43:41,887 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:41,888 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=557913855dcf97d8b784fddf23bec5ee, regionState=CLOSED 2024-11-14T02:43:41,889 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 because future has completed 2024-11-14T02:43:41,892 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-14T02:43:41,892 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 557913855dcf97d8b784fddf23bec5ee, server=83c5a5c119d5,41455,1731552191259 in 1.8790 sec 2024-11-14T02:43:41,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-14T02:43:41,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=557913855dcf97d8b784fddf23bec5ee, UNASSIGN in 1.8870 sec 2024-11-14T02:43:41,896 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-14T02:43:41,896 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8950 sec 2024-11-14T02:43:41,897 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552221897"}]},"ts":"1731552221897"} 2024-11-14T02:43:41,899 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T02:43:41,899 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T02:43:41,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9090 sec 2024-11-14T02:43:42,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-14T02:43:42,145 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:42,149 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T02:43:42,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:43:42,156 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:43:42,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-14T02:43:42,157 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:43:42,162 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:42,167 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A, FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B, FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C, FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/recovered.edits] 2024-11-14T02:43:42,171 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/757e954e5a9641f69df4f851c8517150 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/757e954e5a9641f69df4f851c8517150 2024-11-14T02:43:42,172 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/eb45b2fffcdf425296b9b3d370e77880 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/A/eb45b2fffcdf425296b9b3d370e77880 2024-11-14T02:43:42,176 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/56d6f613ac3748cb82eb9dd918dd3767 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/56d6f613ac3748cb82eb9dd918dd3767 2024-11-14T02:43:42,178 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/71fb27a2b09843319ef1a39457db98d0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/B/71fb27a2b09843319ef1a39457db98d0 2024-11-14T02:43:42,181 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/23f4a52ff08d40da91666de75f7fa8f1 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/23f4a52ff08d40da91666de75f7fa8f1 2024-11-14T02:43:42,183 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/2df9a4364717410bb24d0d8ff4c09d1b to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/2df9a4364717410bb24d0d8ff4c09d1b 2024-11-14T02:43:42,184 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/4f4dbd6bca8b4538a5c69eb29948eaf3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/4f4dbd6bca8b4538a5c69eb29948eaf3 2024-11-14T02:43:42,186 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/8f21c0ffec9342a6a46972a8cd4e1556 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/C/8f21c0ffec9342a6a46972a8cd4e1556 2024-11-14T02:43:42,190 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/recovered.edits/542.seqid to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee/recovered.edits/542.seqid 2024-11-14T02:43:42,190 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/557913855dcf97d8b784fddf23bec5ee 2024-11-14T02:43:42,190 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T02:43:42,195 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:43:42,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-14T02:43:42,204 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T02:43:42,232 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T02:43:42,233 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:43:42,234 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T02:43:42,234 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731552222234"}]},"ts":"9223372036854775807"} 2024-11-14T02:43:42,238 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T02:43:42,238 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 557913855dcf97d8b784fddf23bec5ee, NAME => 'TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T02:43:42,238 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T02:43:42,238 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731552222238"}]},"ts":"9223372036854775807"} 2024-11-14T02:43:42,241 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T02:43:42,242 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:43:42,243 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 92 msec 2024-11-14T02:43:42,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-14T02:43:42,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T02:43:42,265 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:42,277 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=240 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;83c5a5c119d5:41455-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_40334821_22 at /127.0.0.1:50158 [Waiting for operation #342] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-174932997_22 at /127.0.0.1:52610 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-174932997_22 at /127.0.0.1:47536 [Waiting for operation #351] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_40334821_22 at /127.0.0.1:52608 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=308 (was 208) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5491 (was 6071) 2024-11-14T02:43:42,286 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=240, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=308, ProcessCount=11, AvailableMemoryMB=5491 2024-11-14T02:43:42,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T02:43:42,288 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T02:43:42,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T02:43:42,290 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T02:43:42,291 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:42,291 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-14T02:43:42,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-14T02:43:42,292 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T02:43:42,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741950_1126 (size=959) 2024-11-14T02:43:42,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-14T02:43:42,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-14T02:43:42,703 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b 2024-11-14T02:43:42,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741951_1127 (size=53) 2024-11-14T02:43:42,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-14T02:43:43,115 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:43:43,115 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 30ff5d65ef696bee83471a2834e374a0, disabling compactions & flushes 2024-11-14T02:43:43,115 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:43,116 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:43,116 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. after waiting 0 ms 2024-11-14T02:43:43,116 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:43,116 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:43,116 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 30ff5d65ef696bee83471a2834e374a0: Waiting for close lock at 1731552223115Disabling compacts and flushes for region at 1731552223115Disabling writes for close at 1731552223116 (+1 ms)Writing region close event to WAL at 1731552223116Closed at 1731552223116 2024-11-14T02:43:43,120 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T02:43:43,121 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731552223120"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731552223120"}]},"ts":"1731552223120"} 2024-11-14T02:43:43,125 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T02:43:43,127 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T02:43:43,127 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552223127"}]},"ts":"1731552223127"} 2024-11-14T02:43:43,129 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T02:43:43,129 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=30ff5d65ef696bee83471a2834e374a0, ASSIGN}] 2024-11-14T02:43:43,130 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=30ff5d65ef696bee83471a2834e374a0, ASSIGN 2024-11-14T02:43:43,131 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=30ff5d65ef696bee83471a2834e374a0, ASSIGN; state=OFFLINE, location=83c5a5c119d5,41455,1731552191259; forceNewPlan=false, retain=false 2024-11-14T02:43:43,282 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=30ff5d65ef696bee83471a2834e374a0, regionState=OPENING, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:43,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=30ff5d65ef696bee83471a2834e374a0, ASSIGN because future has completed 2024-11-14T02:43:43,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259}] 2024-11-14T02:43:43,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-14T02:43:43,446 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:43,446 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:43:43,446 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:43,446 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:43:43,447 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:43,447 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:43,449 INFO [StoreOpener-30ff5d65ef696bee83471a2834e374a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:43,452 INFO [StoreOpener-30ff5d65ef696bee83471a2834e374a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30ff5d65ef696bee83471a2834e374a0 columnFamilyName A 2024-11-14T02:43:43,452 DEBUG [StoreOpener-30ff5d65ef696bee83471a2834e374a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:43,453 INFO [StoreOpener-30ff5d65ef696bee83471a2834e374a0-1 {}] regionserver.HStore(327): Store=30ff5d65ef696bee83471a2834e374a0/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:43:43,453 INFO [StoreOpener-30ff5d65ef696bee83471a2834e374a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:43,454 INFO [StoreOpener-30ff5d65ef696bee83471a2834e374a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30ff5d65ef696bee83471a2834e374a0 columnFamilyName B 2024-11-14T02:43:43,454 DEBUG [StoreOpener-30ff5d65ef696bee83471a2834e374a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:43,455 INFO [StoreOpener-30ff5d65ef696bee83471a2834e374a0-1 {}] regionserver.HStore(327): Store=30ff5d65ef696bee83471a2834e374a0/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:43:43,455 INFO [StoreOpener-30ff5d65ef696bee83471a2834e374a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:43,456 INFO [StoreOpener-30ff5d65ef696bee83471a2834e374a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30ff5d65ef696bee83471a2834e374a0 columnFamilyName C 2024-11-14T02:43:43,456 DEBUG [StoreOpener-30ff5d65ef696bee83471a2834e374a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:43,456 INFO [StoreOpener-30ff5d65ef696bee83471a2834e374a0-1 {}] regionserver.HStore(327): Store=30ff5d65ef696bee83471a2834e374a0/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:43:43,456 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:43,457 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:43,457 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:43,458 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:43,458 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:43,459 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T02:43:43,460 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:43,462 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T02:43:43,463 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 30ff5d65ef696bee83471a2834e374a0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68148280, jitterRate=0.015488505363464355}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T02:43:43,463 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:43,463 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 30ff5d65ef696bee83471a2834e374a0: Running coprocessor pre-open hook at 1731552223447Writing region info on filesystem at 1731552223447Initializing all the Stores at 1731552223448 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552223448Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552223448Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552223448Cleaning up temporary data from old regions at 1731552223458 (+10 ms)Running coprocessor post-open hooks at 1731552223463 (+5 ms)Region opened successfully at 1731552223463 2024-11-14T02:43:43,464 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., pid=34, masterSystemTime=1731552223442 2024-11-14T02:43:43,466 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:43,466 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:43,466 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=30ff5d65ef696bee83471a2834e374a0, regionState=OPEN, openSeqNum=2, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:43,468 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 because future has completed 2024-11-14T02:43:43,471 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-14T02:43:43,471 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 in 181 msec 2024-11-14T02:43:43,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-14T02:43:43,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=30ff5d65ef696bee83471a2834e374a0, ASSIGN in 342 msec 2024-11-14T02:43:43,474 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T02:43:43,474 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552223474"}]},"ts":"1731552223474"} 2024-11-14T02:43:43,478 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T02:43:43,480 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T02:43:43,482 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1920 sec 2024-11-14T02:43:44,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-14T02:43:44,434 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:44,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bcb2494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:44,435 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:43:44,435 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:44,437 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:43:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:43:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f8fb233, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:44,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:43:44,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:44,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:44,439 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:44,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@147b24c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:44,441 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:43:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:44,444 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:44,446 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T02:43:44,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 83c5a5c119d5,43239,1731552190584 2024-11-14T02:43:44,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@480de345 2024-11-14T02:43:44,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T02:43:44,450 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T02:43:44,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T02:43:44,455 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T02:43:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T02:43:44,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741952_1128 (size=995) 2024-11-14T02:43:44,879 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-14T02:43:44,880 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-14T02:43:44,885 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T02:43:44,893 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=30ff5d65ef696bee83471a2834e374a0, REOPEN/MOVE}] 2024-11-14T02:43:44,893 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=30ff5d65ef696bee83471a2834e374a0, REOPEN/MOVE 2024-11-14T02:43:44,894 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=30ff5d65ef696bee83471a2834e374a0, regionState=CLOSING, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:44,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=30ff5d65ef696bee83471a2834e374a0, REOPEN/MOVE because future has completed 2024-11-14T02:43:44,896 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T02:43:44,896 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259}] 2024-11-14T02:43:45,049 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:45,049 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T02:43:45,050 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 30ff5d65ef696bee83471a2834e374a0, disabling compactions & flushes 2024-11-14T02:43:45,050 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:45,050 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:45,050 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. after waiting 0 ms 2024-11-14T02:43:45,050 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:45,054 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T02:43:45,055 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:45,055 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 30ff5d65ef696bee83471a2834e374a0: Waiting for close lock at 1731552225050Running coprocessor pre-close hooks at 1731552225050Disabling compacts and flushes for region at 1731552225050Disabling writes for close at 1731552225050Writing region close event to WAL at 1731552225050Running coprocessor post-close hooks at 1731552225055 (+5 ms)Closed at 1731552225055 2024-11-14T02:43:45,055 WARN [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 30ff5d65ef696bee83471a2834e374a0 to self. 2024-11-14T02:43:45,057 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:45,057 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=30ff5d65ef696bee83471a2834e374a0, regionState=CLOSED 2024-11-14T02:43:45,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 because future has completed 2024-11-14T02:43:45,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-14T02:43:45,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 in 164 msec 2024-11-14T02:43:45,062 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=30ff5d65ef696bee83471a2834e374a0, REOPEN/MOVE; state=CLOSED, location=83c5a5c119d5,41455,1731552191259; forceNewPlan=false, retain=true 2024-11-14T02:43:45,213 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=30ff5d65ef696bee83471a2834e374a0, regionState=OPENING, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:45,217 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=30ff5d65ef696bee83471a2834e374a0, REOPEN/MOVE because future has completed 2024-11-14T02:43:45,217 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259}] 2024-11-14T02:43:45,374 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:45,374 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:43:45,374 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:45,374 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:43:45,374 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:45,374 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:45,378 INFO [StoreOpener-30ff5d65ef696bee83471a2834e374a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:45,383 INFO [StoreOpener-30ff5d65ef696bee83471a2834e374a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30ff5d65ef696bee83471a2834e374a0 columnFamilyName A 2024-11-14T02:43:45,385 DEBUG [StoreOpener-30ff5d65ef696bee83471a2834e374a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:45,386 INFO [StoreOpener-30ff5d65ef696bee83471a2834e374a0-1 {}] regionserver.HStore(327): Store=30ff5d65ef696bee83471a2834e374a0/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:43:45,386 INFO [StoreOpener-30ff5d65ef696bee83471a2834e374a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:45,387 INFO [StoreOpener-30ff5d65ef696bee83471a2834e374a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30ff5d65ef696bee83471a2834e374a0 columnFamilyName B 2024-11-14T02:43:45,387 DEBUG [StoreOpener-30ff5d65ef696bee83471a2834e374a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:45,387 INFO [StoreOpener-30ff5d65ef696bee83471a2834e374a0-1 {}] regionserver.HStore(327): Store=30ff5d65ef696bee83471a2834e374a0/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:43:45,388 INFO [StoreOpener-30ff5d65ef696bee83471a2834e374a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:45,388 INFO [StoreOpener-30ff5d65ef696bee83471a2834e374a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30ff5d65ef696bee83471a2834e374a0 columnFamilyName C 2024-11-14T02:43:45,388 DEBUG [StoreOpener-30ff5d65ef696bee83471a2834e374a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:45,389 INFO [StoreOpener-30ff5d65ef696bee83471a2834e374a0-1 {}] regionserver.HStore(327): Store=30ff5d65ef696bee83471a2834e374a0/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:43:45,389 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:45,390 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:45,391 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:45,392 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:45,392 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:45,392 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T02:43:45,394 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:45,395 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 30ff5d65ef696bee83471a2834e374a0; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70315638, jitterRate=0.047784656286239624}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T02:43:45,395 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:45,396 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 30ff5d65ef696bee83471a2834e374a0: Running coprocessor pre-open hook at 1731552225374Writing region info on filesystem at 1731552225374Initializing all the Stores at 1731552225375 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552225375Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552225375Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552225375Cleaning up temporary data from old regions at 1731552225392 (+17 ms)Running coprocessor post-open hooks at 1731552225395 (+3 ms)Region opened successfully at 1731552225396 (+1 ms) 2024-11-14T02:43:45,397 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., pid=39, masterSystemTime=1731552225370 2024-11-14T02:43:45,399 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:45,399 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:45,399 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=30ff5d65ef696bee83471a2834e374a0, regionState=OPEN, openSeqNum=5, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:45,401 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 because future has completed 2024-11-14T02:43:45,404 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-14T02:43:45,404 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 in 185 msec 2024-11-14T02:43:45,406 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-14T02:43:45,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=30ff5d65ef696bee83471a2834e374a0, REOPEN/MOVE in 511 msec 2024-11-14T02:43:45,411 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-14T02:43:45,411 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 524 msec 2024-11-14T02:43:45,415 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 954 msec 2024-11-14T02:43:45,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-14T02:43:45,417 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:45,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ed5d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:45,418 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:43:45,418 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:45,419 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:43:45,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:45,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:43:45,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15700b49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:45,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:43:45,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:45,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:45,422 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:45,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554d2b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:45,423 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4406a4b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:45,423 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:43:45,423 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:45,424 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:43:45,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:45,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:43:45,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1389b68a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:45,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:43:45,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:45,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:45,426 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:45,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e903769, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:45,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d0a90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:45,428 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:43:45,428 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:45,429 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:43:45,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:45,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:43:45,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49310590, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:45,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:43:45,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:45,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:45,431 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:45,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66595884, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:45,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b236c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:45,432 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:43:45,432 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:45,433 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:43:45,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:45,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:43:45,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59fdda83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:45,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:43:45,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:45,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:45,435 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:45,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32af672a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:45,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26dcfede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:45,436 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:43:45,436 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:45,437 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:43:45,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:45,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:43:45,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c8d5a43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:45,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:43:45,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:45,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:45,439 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:45,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@520cf6c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:45,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ae3464, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:45,440 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:43:45,440 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:45,441 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:43:45,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:45,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:43:45,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@94fac08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:45,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:43:45,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:45,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:45,443 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:45,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@623f8114, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:45,444 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bbb11d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:45,444 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:43:45,444 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:45,445 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:43:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:43:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5308acec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:43:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:45,447 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b08d39c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:45,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d2a6fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:45,448 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:43:45,448 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:45,449 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:43:45,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:45,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:43:45,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef62b6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:45,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:43:45,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:45,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:45,450 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:45,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70104d17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:45,451 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30ade04f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:45,451 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:43:45,451 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:45,452 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:43:45,452 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:45,452 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:43:45,452 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e19aaf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:45,452 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:43:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:45,454 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a87beba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:45,455 DEBUG [Thread-642 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:45,455 DEBUG [Thread-648 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:45,455 DEBUG [Thread-657 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:45,455 DEBUG [Thread-646 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:45,456 DEBUG [Thread-659 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:45,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:43:45,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:43:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:43:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:43:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:45,457 DEBUG [Thread-657 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:45,457 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:43:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:43:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:45,458 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:45,458 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:45,458 DEBUG [Thread-653 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:45,458 DEBUG [Thread-655 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:45,458 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45930, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:45,459 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:45,459 DEBUG [Thread-644 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:45,459 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:45,460 DEBUG [Thread-653 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:43:45,460 DEBUG [Thread-655 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:43:45,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:43:45,460 DEBUG [Thread-650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:45,460 DEBUG [Thread-653 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:45,460 DEBUG [Thread-655 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:45,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:45,461 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:45,461 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:45,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:43:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:43:45,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:43:45,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:43:45,464 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:45,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:43:45,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:43:45,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:43:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:43:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:43:45,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:43:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-14T02:43:45,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T02:43:45,493 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:43:45,494 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:43:45,495 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:43:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:45,503 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:43:45,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30ff5d65ef696bee83471a2834e374a0 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-14T02:43:45,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:45,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552285547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:45,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:45,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552285549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:45,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552285551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:45,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:45,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:45,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552285554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:45,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:45,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:45,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552285555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:45,562 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,562 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,562 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:45,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111471a1d8a101ce4a8188b4e32331eb8a2e_30ff5d65ef696bee83471a2834e374a0 is 50, key is test_row_0/A:col10/1731552225501/Put/seqid=0 2024-11-14T02:43:45,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741953_1129 (size=12154) 2024-11-14T02:43:45,578 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:45,586 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111471a1d8a101ce4a8188b4e32331eb8a2e_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111471a1d8a101ce4a8188b4e32331eb8a2e_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:45,588 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/2683c858f1d04f06ab5e954875466f14, store: [table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:45,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/2683c858f1d04f06ab5e954875466f14 is 175, key is test_row_0/A:col10/1731552225501/Put/seqid=0 2024-11-14T02:43:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T02:43:45,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741954_1130 (size=30955) 2024-11-14T02:43:45,609 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/2683c858f1d04f06ab5e954875466f14 2024-11-14T02:43:45,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/fafbd5da692241dca9b56739ffbca99f is 50, key is test_row_0/B:col10/1731552225501/Put/seqid=0 2024-11-14T02:43:45,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T02:43:45,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:45,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:43:45,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:45,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:45,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:45,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552285669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:45,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:45,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552285670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:45,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:45,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552285671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:45,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552285671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:45,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:45,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:45,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552285672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:45,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741955_1131 (size=12001) 2024-11-14T02:43:45,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:45,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T02:43:45,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:45,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:43:45,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:45,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:45,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:45,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:45,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T02:43:45,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:45,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552285884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552285884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552285884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552285884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552285884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:45,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T02:43:45,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:45,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:43:45,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:45,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:45,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:45,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:46,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/fafbd5da692241dca9b56739ffbca99f 2024-11-14T02:43:46,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/048f67ce9a8b46d697bd0d90fb1efb04 is 50, key is test_row_0/C:col10/1731552225501/Put/seqid=0 2024-11-14T02:43:46,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741956_1132 (size=12001) 2024-11-14T02:43:46,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/048f67ce9a8b46d697bd0d90fb1efb04 2024-11-14T02:43:46,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T02:43:46,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:46,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:43:46,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:46,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:46,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:46,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/2683c858f1d04f06ab5e954875466f14 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/2683c858f1d04f06ab5e954875466f14 2024-11-14T02:43:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T02:43:46,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/2683c858f1d04f06ab5e954875466f14, entries=150, sequenceid=17, filesize=30.2 K 2024-11-14T02:43:46,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/fafbd5da692241dca9b56739ffbca99f as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/fafbd5da692241dca9b56739ffbca99f 2024-11-14T02:43:46,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/fafbd5da692241dca9b56739ffbca99f, entries=150, sequenceid=17, filesize=11.7 K 2024-11-14T02:43:46,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/048f67ce9a8b46d697bd0d90fb1efb04 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/048f67ce9a8b46d697bd0d90fb1efb04 2024-11-14T02:43:46,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/048f67ce9a8b46d697bd0d90fb1efb04, entries=150, sequenceid=17, filesize=11.7 K 2024-11-14T02:43:46,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 30ff5d65ef696bee83471a2834e374a0 in 672ms, sequenceid=17, compaction requested=false 2024-11-14T02:43:46,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:43:46,199 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:43:46,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30ff5d65ef696bee83471a2834e374a0 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T02:43:46,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:46,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e13607ec63ba4011afad6e9115ea070f_30ff5d65ef696bee83471a2834e374a0 is 50, key is test_row_0/A:col10/1731552226199/Put/seqid=0 2024-11-14T02:43:46,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:46,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552286215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:46,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:46,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:46,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552286217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:46,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:46,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552286217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552286219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:46,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552286220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:46,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741957_1133 (size=17034) 2024-11-14T02:43:46,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:46,235 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:46,241 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e13607ec63ba4011afad6e9115ea070f_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e13607ec63ba4011afad6e9115ea070f_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:46,242 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/088e7c2877494f00bd2227feb036e915, store: [table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:46,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/088e7c2877494f00bd2227feb036e915 is 175, key is test_row_0/A:col10/1731552226199/Put/seqid=0 2024-11-14T02:43:46,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741958_1134 (size=48139) 2024-11-14T02:43:46,259 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=43, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/088e7c2877494f00bd2227feb036e915 2024-11-14T02:43:46,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T02:43:46,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:46,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:43:46,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:46,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:46,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:46,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/3efdfd8e34b44e53a6c7fb40491ded01 is 50, key is test_row_0/B:col10/1731552226199/Put/seqid=0 2024-11-14T02:43:46,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741959_1135 (size=12001) 2024-11-14T02:43:46,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/3efdfd8e34b44e53a6c7fb40491ded01 2024-11-14T02:43:46,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/20c21f212c6a47348cced829fce0e398 is 50, key is test_row_0/C:col10/1731552226199/Put/seqid=0 2024-11-14T02:43:46,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741960_1136 (size=12001) 2024-11-14T02:43:46,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/20c21f212c6a47348cced829fce0e398 2024-11-14T02:43:46,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/088e7c2877494f00bd2227feb036e915 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/088e7c2877494f00bd2227feb036e915 2024-11-14T02:43:46,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/088e7c2877494f00bd2227feb036e915, entries=250, sequenceid=43, filesize=47.0 K 2024-11-14T02:43:46,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/3efdfd8e34b44e53a6c7fb40491ded01 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/3efdfd8e34b44e53a6c7fb40491ded01 2024-11-14T02:43:46,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552286333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:46,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/3efdfd8e34b44e53a6c7fb40491ded01, entries=150, sequenceid=43, filesize=11.7 K 2024-11-14T02:43:46,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/20c21f212c6a47348cced829fce0e398 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/20c21f212c6a47348cced829fce0e398 2024-11-14T02:43:46,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552286344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:46,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/20c21f212c6a47348cced829fce0e398, entries=150, sequenceid=43, filesize=11.7 K 2024-11-14T02:43:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:46,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 30ff5d65ef696bee83471a2834e374a0 in 154ms, sequenceid=43, compaction requested=false 2024-11-14T02:43:46,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:43:46,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:46,354 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:43:46,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30ff5d65ef696bee83471a2834e374a0 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T02:43:46,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111404aea900beef4fc7994f1cb39572b539_30ff5d65ef696bee83471a2834e374a0 is 50, key is test_row_0/A:col10/1731552226353/Put/seqid=0 2024-11-14T02:43:46,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741961_1137 (size=19474) 2024-11-14T02:43:46,405 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:46,417 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111404aea900beef4fc7994f1cb39572b539_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111404aea900beef4fc7994f1cb39572b539_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:46,419 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/db500ab5e1df494d86929c4ab87e28c1, store: [table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:46,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/db500ab5e1df494d86929c4ab87e28c1 is 175, key is test_row_0/A:col10/1731552226353/Put/seqid=0 2024-11-14T02:43:46,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T02:43:46,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:46,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:43:46,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:46,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:46,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:46,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741962_1138 (size=56733) 2024-11-14T02:43:46,438 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=57, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/db500ab5e1df494d86929c4ab87e28c1 2024-11-14T02:43:46,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552286438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:46,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552286440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:46,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:46,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:46,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552286448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:46,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/c33442bae8354bfcb82cec73c0fc4762 is 50, key is test_row_0/B:col10/1731552226353/Put/seqid=0 2024-11-14T02:43:46,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:46,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741963_1139 (size=12001) 2024-11-14T02:43:46,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/c33442bae8354bfcb82cec73c0fc4762 2024-11-14T02:43:46,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/8310e61eb9a94e8a87f09ad8f26a18c9 is 50, key is test_row_0/C:col10/1731552226353/Put/seqid=0 2024-11-14T02:43:46,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741964_1140 (size=12001) 2024-11-14T02:43:46,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/8310e61eb9a94e8a87f09ad8f26a18c9 2024-11-14T02:43:46,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/db500ab5e1df494d86929c4ab87e28c1 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/db500ab5e1df494d86929c4ab87e28c1 2024-11-14T02:43:46,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/db500ab5e1df494d86929c4ab87e28c1, entries=300, sequenceid=57, filesize=55.4 K 2024-11-14T02:43:46,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/c33442bae8354bfcb82cec73c0fc4762 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/c33442bae8354bfcb82cec73c0fc4762 2024-11-14T02:43:46,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/c33442bae8354bfcb82cec73c0fc4762, entries=150, sequenceid=57, filesize=11.7 K 2024-11-14T02:43:46,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/8310e61eb9a94e8a87f09ad8f26a18c9 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/8310e61eb9a94e8a87f09ad8f26a18c9 2024-11-14T02:43:46,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/8310e61eb9a94e8a87f09ad8f26a18c9, entries=150, sequenceid=57, filesize=11.7 K 2024-11-14T02:43:46,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 30ff5d65ef696bee83471a2834e374a0 in 187ms, sequenceid=57, compaction requested=true 2024-11-14T02:43:46,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:43:46,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30ff5d65ef696bee83471a2834e374a0:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:43:46,542 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:46,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:46,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30ff5d65ef696bee83471a2834e374a0:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:43:46,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:46,542 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:46,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30ff5d65ef696bee83471a2834e374a0:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:43:46,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:46,543 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:46,543 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 30ff5d65ef696bee83471a2834e374a0/B is initiating minor compaction (all files) 2024-11-14T02:43:46,543 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30ff5d65ef696bee83471a2834e374a0/B in TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:46,543 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/fafbd5da692241dca9b56739ffbca99f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/3efdfd8e34b44e53a6c7fb40491ded01, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/c33442bae8354bfcb82cec73c0fc4762] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp, totalSize=35.2 K 2024-11-14T02:43:46,544 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting fafbd5da692241dca9b56739ffbca99f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731552225495 2024-11-14T02:43:46,545 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 3efdfd8e34b44e53a6c7fb40491ded01, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731552225550 2024-11-14T02:43:46,545 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 135827 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:46,545 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 30ff5d65ef696bee83471a2834e374a0/A is initiating minor compaction (all files) 2024-11-14T02:43:46,545 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30ff5d65ef696bee83471a2834e374a0/A in TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:46,545 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting c33442bae8354bfcb82cec73c0fc4762, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731552226212 2024-11-14T02:43:46,545 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/2683c858f1d04f06ab5e954875466f14, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/088e7c2877494f00bd2227feb036e915, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/db500ab5e1df494d86929c4ab87e28c1] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp, totalSize=132.6 K 2024-11-14T02:43:46,545 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:46,545 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. files: [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/2683c858f1d04f06ab5e954875466f14, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/088e7c2877494f00bd2227feb036e915, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/db500ab5e1df494d86929c4ab87e28c1] 2024-11-14T02:43:46,551 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2683c858f1d04f06ab5e954875466f14, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731552225495 2024-11-14T02:43:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:46,551 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:43:46,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30ff5d65ef696bee83471a2834e374a0 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-14T02:43:46,551 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 088e7c2877494f00bd2227feb036e915, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731552225539 2024-11-14T02:43:46,552 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting db500ab5e1df494d86929c4ab87e28c1, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731552226212 2024-11-14T02:43:46,556 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30ff5d65ef696bee83471a2834e374a0#B#compaction#122 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:46,556 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/0d294997d59e4f1b85abb2304e47c74e is 50, key is test_row_0/B:col10/1731552226353/Put/seqid=0 2024-11-14T02:43:46,573 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:46,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:46,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552286569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:46,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:46,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552286571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:46,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:46,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:46,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552286573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:46,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:46,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552286576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:46,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:46,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d460ae4c3496404a9aedc61d9b9dc661_30ff5d65ef696bee83471a2834e374a0 is 50, key is test_row_0/A:col10/1731552226421/Put/seqid=0 2024-11-14T02:43:46,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:46,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T02:43:46,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:46,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:46,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:43:46,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:46,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:46,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:46,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552286578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:46,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741965_1141 (size=12104) 2024-11-14T02:43:46,609 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114e39b7ca7eb7642e9b754da02aa8e3ed3_30ff5d65ef696bee83471a2834e374a0 store=[table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:46,618 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/0d294997d59e4f1b85abb2304e47c74e as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/0d294997d59e4f1b85abb2304e47c74e 2024-11-14T02:43:46,623 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114e39b7ca7eb7642e9b754da02aa8e3ed3_30ff5d65ef696bee83471a2834e374a0, store=[table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:46,625 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e39b7ca7eb7642e9b754da02aa8e3ed3_30ff5d65ef696bee83471a2834e374a0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:46,629 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30ff5d65ef696bee83471a2834e374a0/B of 30ff5d65ef696bee83471a2834e374a0 into 0d294997d59e4f1b85abb2304e47c74e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:46,629 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:43:46,629 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., storeName=30ff5d65ef696bee83471a2834e374a0/B, priority=13, startTime=1731552226542; duration=0sec 2024-11-14T02:43:46,629 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:46,629 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30ff5d65ef696bee83471a2834e374a0:B 2024-11-14T02:43:46,629 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:46,631 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:46,632 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 30ff5d65ef696bee83471a2834e374a0/C is initiating minor compaction (all files) 2024-11-14T02:43:46,632 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30ff5d65ef696bee83471a2834e374a0/C in TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:46,632 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/048f67ce9a8b46d697bd0d90fb1efb04, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/20c21f212c6a47348cced829fce0e398, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/8310e61eb9a94e8a87f09ad8f26a18c9] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp, totalSize=35.2 K 2024-11-14T02:43:46,633 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 048f67ce9a8b46d697bd0d90fb1efb04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731552225495 2024-11-14T02:43:46,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T02:43:46,634 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 20c21f212c6a47348cced829fce0e398, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731552225550 2024-11-14T02:43:46,635 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 8310e61eb9a94e8a87f09ad8f26a18c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731552226212 2024-11-14T02:43:46,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741966_1142 (size=14594) 2024-11-14T02:43:46,666 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30ff5d65ef696bee83471a2834e374a0#C#compaction#125 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:46,667 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/1bfc5541fe794f13b4fcec03eee8d40a is 50, key is test_row_0/C:col10/1731552226353/Put/seqid=0 2024-11-14T02:43:46,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741967_1143 (size=4469) 2024-11-14T02:43:46,686 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30ff5d65ef696bee83471a2834e374a0#A#compaction#123 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:46,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552286684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:46,687 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/1fa24a5d86504c85b8feb961ace0ee4d is 175, key is test_row_0/A:col10/1731552226353/Put/seqid=0 2024-11-14T02:43:46,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:46,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:46,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552286684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:46,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:46,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741968_1144 (size=12104) 2024-11-14T02:43:46,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:46,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552286694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:46,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:46,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:46,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552286694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:46,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:46,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552286694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:46,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:46,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:46,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741969_1145 (size=31058) 2024-11-14T02:43:46,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T02:43:46,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:46,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:43:46,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:46,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:46,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:46,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:46,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T02:43:46,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:46,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:43:46,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:46,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:46,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:46,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:46,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:46,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552286894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:46,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:46,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:46,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552286894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:46,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:46,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:46,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552286903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:46,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:46,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:46,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552286903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:46,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:46,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552286903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:46,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:46,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:47,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T02:43:47,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:47,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:43:47,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:47,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:47,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:47,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:47,058 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:47,063 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d460ae4c3496404a9aedc61d9b9dc661_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d460ae4c3496404a9aedc61d9b9dc661_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:47,064 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/0a11afe80ff64a3086639ac7b3d07101, store: [table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:47,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/0a11afe80ff64a3086639ac7b3d07101 is 175, key is test_row_0/A:col10/1731552226421/Put/seqid=0 2024-11-14T02:43:47,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741970_1146 (size=39549) 2024-11-14T02:43:47,069 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/0a11afe80ff64a3086639ac7b3d07101 2024-11-14T02:43:47,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/99b57bf902834728b3a0390c1276b68d is 50, key is test_row_0/B:col10/1731552226421/Put/seqid=0 2024-11-14T02:43:47,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741971_1147 (size=12001) 2024-11-14T02:43:47,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/99b57bf902834728b3a0390c1276b68d 2024-11-14T02:43:47,110 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/1bfc5541fe794f13b4fcec03eee8d40a as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/1bfc5541fe794f13b4fcec03eee8d40a 2024-11-14T02:43:47,113 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/1fa24a5d86504c85b8feb961ace0ee4d as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/1fa24a5d86504c85b8feb961ace0ee4d 2024-11-14T02:43:47,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/eb84132149704060ad64d038d599fec1 is 50, key is test_row_0/C:col10/1731552226421/Put/seqid=0 2024-11-14T02:43:47,117 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30ff5d65ef696bee83471a2834e374a0/C of 30ff5d65ef696bee83471a2834e374a0 into 1bfc5541fe794f13b4fcec03eee8d40a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:47,117 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:43:47,117 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., storeName=30ff5d65ef696bee83471a2834e374a0/C, priority=13, startTime=1731552226542; duration=0sec 2024-11-14T02:43:47,117 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:47,117 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30ff5d65ef696bee83471a2834e374a0:C 2024-11-14T02:43:47,124 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30ff5d65ef696bee83471a2834e374a0/A of 30ff5d65ef696bee83471a2834e374a0 into 1fa24a5d86504c85b8feb961ace0ee4d(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:47,124 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:43:47,124 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., storeName=30ff5d65ef696bee83471a2834e374a0/A, priority=13, startTime=1731552226541; duration=0sec 2024-11-14T02:43:47,124 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:47,124 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30ff5d65ef696bee83471a2834e374a0:A 2024-11-14T02:43:47,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741972_1148 (size=12001) 2024-11-14T02:43:47,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T02:43:47,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:47,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:43:47,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:47,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:47,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:47,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:47,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:47,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552287203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:47,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:47,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552287203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:47,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:47,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:47,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:47,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552287213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:47,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:47,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552287214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:47,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:47,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:47,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552287214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:47,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:47,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,216 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:47,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T02:43:47,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:47,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:43:47,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:47,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:47,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:47,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:47,422 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T02:43:47,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T02:43:47,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:47,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:43:47,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:47,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:47,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:47,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:47,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/eb84132149704060ad64d038d599fec1 2024-11-14T02:43:47,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/0a11afe80ff64a3086639ac7b3d07101 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/0a11afe80ff64a3086639ac7b3d07101 2024-11-14T02:43:47,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/0a11afe80ff64a3086639ac7b3d07101, entries=200, sequenceid=80, filesize=38.6 K 2024-11-14T02:43:47,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/99b57bf902834728b3a0390c1276b68d as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/99b57bf902834728b3a0390c1276b68d 2024-11-14T02:43:47,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/99b57bf902834728b3a0390c1276b68d, entries=150, sequenceid=80, filesize=11.7 K 2024-11-14T02:43:47,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/eb84132149704060ad64d038d599fec1 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/eb84132149704060ad64d038d599fec1 2024-11-14T02:43:47,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/eb84132149704060ad64d038d599fec1, entries=150, sequenceid=80, filesize=11.7 K 2024-11-14T02:43:47,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 30ff5d65ef696bee83471a2834e374a0 in 1003ms, sequenceid=80, compaction requested=false 2024-11-14T02:43:47,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:43:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T02:43:47,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T02:43:47,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:47,660 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 30ff5d65ef696bee83471a2834e374a0 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-14T02:43:47,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114aa1a19e298d144bc85c4e80bb06ce237_30ff5d65ef696bee83471a2834e374a0 is 50, key is test_row_0/A:col10/1731552226559/Put/seqid=0 2024-11-14T02:43:47,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741973_1149 (size=12154) 2024-11-14T02:43:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:47,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:43:47,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552287736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:47,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:47,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552287737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:47,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:47,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552287738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:47,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552287741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:47,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:47,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552287742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:47,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:47,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:47,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:47,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:47,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552287854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:47,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552287854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:47,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:47,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:47,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552287854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:47,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552287854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:47,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552287854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:47,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:47,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:47,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:47,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:47,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:48,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:48,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:48,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552288063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:48,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552288063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:48,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:48,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:48,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:48,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552288063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:48,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552288063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:48,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:48,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:48,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552288063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:48,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:48,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:48,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:48,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:48,094 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114aa1a19e298d144bc85c4e80bb06ce237_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114aa1a19e298d144bc85c4e80bb06ce237_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:48,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/f2462c25f9314afb8b8151fad5265132, store: [table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:48,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/f2462c25f9314afb8b8151fad5265132 is 175, key is test_row_0/A:col10/1731552226559/Put/seqid=0 2024-11-14T02:43:48,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741974_1150 (size=30955) 2024-11-14T02:43:48,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:48,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552288373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:48,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:48,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552288373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:48,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:48,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:48,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552288373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:48,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:48,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:48,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552288373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:48,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:48,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552288373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:48,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:48,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:48,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:48,503 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/f2462c25f9314afb8b8151fad5265132 2024-11-14T02:43:48,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/ab1e6843dca245908288860aa03ed29e is 50, key is test_row_0/B:col10/1731552226559/Put/seqid=0 2024-11-14T02:43:48,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741975_1151 (size=12001) 2024-11-14T02:43:48,516 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/ab1e6843dca245908288860aa03ed29e 2024-11-14T02:43:48,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/1fde2989bfb443d09fced3c3552abcc6 is 50, key is test_row_0/C:col10/1731552226559/Put/seqid=0 2024-11-14T02:43:48,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741976_1152 (size=12001) 2024-11-14T02:43:48,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:48,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552288883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:48,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:48,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552288883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:48,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:48,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:48,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:48,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552288884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:48,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:48,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552288884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:48,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:48,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:48,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:48,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552288884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:48,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:48,929 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/1fde2989bfb443d09fced3c3552abcc6 2024-11-14T02:43:48,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/f2462c25f9314afb8b8151fad5265132 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/f2462c25f9314afb8b8151fad5265132 2024-11-14T02:43:48,940 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/f2462c25f9314afb8b8151fad5265132, entries=150, sequenceid=96, filesize=30.2 K 2024-11-14T02:43:48,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/ab1e6843dca245908288860aa03ed29e as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/ab1e6843dca245908288860aa03ed29e 2024-11-14T02:43:48,946 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/ab1e6843dca245908288860aa03ed29e, entries=150, sequenceid=96, filesize=11.7 K 2024-11-14T02:43:48,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/1fde2989bfb443d09fced3c3552abcc6 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/1fde2989bfb443d09fced3c3552abcc6 2024-11-14T02:43:48,955 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/1fde2989bfb443d09fced3c3552abcc6, entries=150, sequenceid=96, filesize=11.7 K 2024-11-14T02:43:48,956 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 30ff5d65ef696bee83471a2834e374a0 in 1296ms, sequenceid=96, compaction requested=true 2024-11-14T02:43:48,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:43:48,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:48,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-14T02:43:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-14T02:43:48,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-14T02:43:48,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4640 sec 2024-11-14T02:43:48,961 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 3.5030 sec 2024-11-14T02:43:49,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T02:43:49,654 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:49,654 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:43:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-14T02:43:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T02:43:49,656 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:43:49,657 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:43:49,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:43:49,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T02:43:49,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T02:43:49,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:49,809 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 30ff5d65ef696bee83471a2834e374a0 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-14T02:43:49,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ea47234ca5f3465bb7fd9dacb5ab1c39_30ff5d65ef696bee83471a2834e374a0 is 50, key is test_row_0/A:col10/1731552227735/Put/seqid=0 2024-11-14T02:43:49,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741977_1153 (size=12154) 2024-11-14T02:43:49,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:49,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:43:49,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:49,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552289903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552289905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:49,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552289905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:49,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552289906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:49,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:49,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552289907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:49,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:49,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:49,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T02:43:50,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:50,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552290013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:50,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:50,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552290013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:50,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:50,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:50,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552290013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:50,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552290013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:50,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:50,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:50,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552290013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:50,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:50,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:50,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:50,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552290223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:50,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:50,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:50,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552290223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:50,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:50,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:50,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:50,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552290223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:50,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:50,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:50,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552290223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:50,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:50,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:50,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552290223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:50,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:50,232 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ea47234ca5f3465bb7fd9dacb5ab1c39_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ea47234ca5f3465bb7fd9dacb5ab1c39_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:50,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/dca56e5935e94b749d15f528727fd627, store: [table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:50,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/dca56e5935e94b749d15f528727fd627 is 175, key is test_row_0/A:col10/1731552227735/Put/seqid=0 2024-11-14T02:43:50,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741978_1154 (size=30955) 2024-11-14T02:43:50,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T02:43:50,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:50,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552290533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:50,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:50,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:50,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552290533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:50,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:50,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:50,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552290533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:50,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:50,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:50,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552290533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:50,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:50,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:50,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552290533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:50,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:50,666 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/dca56e5935e94b749d15f528727fd627 2024-11-14T02:43:50,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/80628c31304d46baa19157487156ab75 is 50, key is test_row_0/B:col10/1731552227735/Put/seqid=0 2024-11-14T02:43:50,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741979_1155 (size=12001) 2024-11-14T02:43:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T02:43:51,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:51,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552291043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552291043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:51,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:51,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:51,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552291053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:51,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:51,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552291053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:51,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:51,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:51,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552291054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:51,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:51,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:51,086 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/80628c31304d46baa19157487156ab75 2024-11-14T02:43:51,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/e95d4817c038457dadf2a7620911821c is 50, key is test_row_0/C:col10/1731552227735/Put/seqid=0 2024-11-14T02:43:51,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741980_1156 (size=12001) 2024-11-14T02:43:51,499 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/e95d4817c038457dadf2a7620911821c 2024-11-14T02:43:51,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/dca56e5935e94b749d15f528727fd627 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/dca56e5935e94b749d15f528727fd627 2024-11-14T02:43:51,510 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/dca56e5935e94b749d15f528727fd627, entries=150, sequenceid=119, filesize=30.2 K 2024-11-14T02:43:51,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/80628c31304d46baa19157487156ab75 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/80628c31304d46baa19157487156ab75 2024-11-14T02:43:51,520 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/80628c31304d46baa19157487156ab75, entries=150, sequenceid=119, filesize=11.7 K 2024-11-14T02:43:51,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/e95d4817c038457dadf2a7620911821c as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/e95d4817c038457dadf2a7620911821c 2024-11-14T02:43:51,527 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/e95d4817c038457dadf2a7620911821c, entries=150, sequenceid=119, filesize=11.7 K 2024-11-14T02:43:51,528 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 30ff5d65ef696bee83471a2834e374a0 in 1719ms, sequenceid=119, compaction requested=true 2024-11-14T02:43:51,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:43:51,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:51,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-14T02:43:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-14T02:43:51,532 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-14T02:43:51,532 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8730 sec 2024-11-14T02:43:51,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.8790 sec 2024-11-14T02:43:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T02:43:51,803 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:51,804 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:43:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-14T02:43:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T02:43:51,806 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:43:51,807 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:43:51,807 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:43:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T02:43:51,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T02:43:51,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:51,961 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 30ff5d65ef696bee83471a2834e374a0 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-14T02:43:51,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143a27086761e7404396327c62c3aaabcb_30ff5d65ef696bee83471a2834e374a0 is 50, key is test_row_0/A:col10/1731552229904/Put/seqid=0 2024-11-14T02:43:51,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741981_1157 (size=12304) 2024-11-14T02:43:51,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:51,982 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143a27086761e7404396327c62c3aaabcb_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143a27086761e7404396327c62c3aaabcb_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:51,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/52c1bfba06c547398b0d663070eeb0ca, store: [table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:51,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/52c1bfba06c547398b0d663070eeb0ca is 175, key is test_row_0/A:col10/1731552229904/Put/seqid=0 2024-11-14T02:43:51,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741982_1158 (size=31105) 2024-11-14T02:43:52,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:52,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:43:52,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:52,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552292116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:52,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:52,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T02:43:52,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552292116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:52,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552292117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:52,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552292118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:52,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552292118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:52,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552292223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:52,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:52,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552292233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:52,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:52,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552292234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:52,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:52,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:52,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552292234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:52,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:52,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:52,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552292234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:52,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:52,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:52,392 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/52c1bfba06c547398b0d663070eeb0ca 2024-11-14T02:43:52,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/b0988fe5cc704184b40d2084937e8e28 is 50, key is test_row_0/B:col10/1731552229904/Put/seqid=0 2024-11-14T02:43:52,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741983_1159 (size=12151) 2024-11-14T02:43:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T02:43:52,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552292433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:52,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552292444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:52,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:52,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552292444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:52,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552292444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:52,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552292444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:52,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552292743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:52,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:52,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552292763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:52,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552292764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:52,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:52,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552292764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:52,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:52,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552292764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:52,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:52,810 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/b0988fe5cc704184b40d2084937e8e28 2024-11-14T02:43:52,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/f952d17a299c4c78b13a08c7e437127c is 50, key is test_row_0/C:col10/1731552229904/Put/seqid=0 2024-11-14T02:43:52,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741984_1160 (size=12151) 2024-11-14T02:43:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T02:43:53,223 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/f952d17a299c4c78b13a08c7e437127c 2024-11-14T02:43:53,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/52c1bfba06c547398b0d663070eeb0ca as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/52c1bfba06c547398b0d663070eeb0ca 2024-11-14T02:43:53,233 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/52c1bfba06c547398b0d663070eeb0ca, entries=150, sequenceid=132, filesize=30.4 K 2024-11-14T02:43:53,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/b0988fe5cc704184b40d2084937e8e28 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/b0988fe5cc704184b40d2084937e8e28 2024-11-14T02:43:53,240 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/b0988fe5cc704184b40d2084937e8e28, entries=150, sequenceid=132, filesize=11.9 K 2024-11-14T02:43:53,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/f952d17a299c4c78b13a08c7e437127c as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/f952d17a299c4c78b13a08c7e437127c 2024-11-14T02:43:53,246 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/f952d17a299c4c78b13a08c7e437127c, entries=150, sequenceid=132, filesize=11.9 K 2024-11-14T02:43:53,247 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 30ff5d65ef696bee83471a2834e374a0 in 1287ms, sequenceid=132, compaction requested=true 2024-11-14T02:43:53,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:43:53,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:53,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-14T02:43:53,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-14T02:43:53,250 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-14T02:43:53,250 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4410 sec 2024-11-14T02:43:53,252 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.4470 sec 2024-11-14T02:43:53,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:53,258 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:43:53,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30ff5d65ef696bee83471a2834e374a0 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-14T02:43:53,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145b98e00d45034f3e8638238d4ef5f6fd_30ff5d65ef696bee83471a2834e374a0 is 50, key is test_row_0/A:col10/1731552232115/Put/seqid=0 2024-11-14T02:43:53,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741985_1161 (size=14794) 2024-11-14T02:43:53,272 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:53,278 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145b98e00d45034f3e8638238d4ef5f6fd_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145b98e00d45034f3e8638238d4ef5f6fd_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:53,279 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/551a9a4ae4ec4dfc82539592d62f0532, store: [table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:53,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/551a9a4ae4ec4dfc82539592d62f0532 is 175, key is test_row_0/A:col10/1731552232115/Put/seqid=0 2024-11-14T02:43:53,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741986_1162 (size=39749) 2024-11-14T02:43:53,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:53,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:53,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552293279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:53,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552293277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:53,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:53,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552293280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:53,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:53,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:53,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:53,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:53,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552293279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:53,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:53,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:53,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552293285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:53,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:53,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552293393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:53,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552293393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552293393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:53,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552293393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:53,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552293394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:53,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:53,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552293603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:53,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552293604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:53,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552293604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:53,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552293604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:53,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552293604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:53,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:53,684 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/551a9a4ae4ec4dfc82539592d62f0532 2024-11-14T02:43:53,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/35e8c183292741ffb5411427d08a3ea1 is 50, key is test_row_0/B:col10/1731552232115/Put/seqid=0 2024-11-14T02:43:53,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741987_1163 (size=12151) 2024-11-14T02:43:53,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552293913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:53,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:53,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552293913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:53,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:53,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552293913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552293913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:53,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552293913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:53,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T02:43:53,953 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:53,954 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:43:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-14T02:43:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T02:43:53,955 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:43:53,956 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:43:53,956 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:43:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T02:43:54,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/35e8c183292741ffb5411427d08a3ea1 2024-11-14T02:43:54,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/4fcc4358ef834ff2935e99ac5be9b43c is 50, key is test_row_0/C:col10/1731552232115/Put/seqid=0 2024-11-14T02:43:54,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T02:43:54,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741988_1164 (size=12151) 2024-11-14T02:43:54,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:54,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:43:54,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:54,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:54,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:54,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T02:43:54,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:54,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:43:54,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:54,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:54,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T02:43:54,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T02:43:54,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:54,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:43:54,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:54,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:54,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:54,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:54,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552294423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:54,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:54,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:54,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552294423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:54,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:54,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:54,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:54,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:54,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552294423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:54,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:54,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:54,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:54,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552294423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:54,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:54,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:54,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:54,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552294433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:54,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:54,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:54,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:54,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/4fcc4358ef834ff2935e99ac5be9b43c 2024-11-14T02:43:54,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/551a9a4ae4ec4dfc82539592d62f0532 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/551a9a4ae4ec4dfc82539592d62f0532 2024-11-14T02:43:54,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/551a9a4ae4ec4dfc82539592d62f0532, entries=200, sequenceid=156, filesize=38.8 K 2024-11-14T02:43:54,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/35e8c183292741ffb5411427d08a3ea1 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/35e8c183292741ffb5411427d08a3ea1 2024-11-14T02:43:54,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/35e8c183292741ffb5411427d08a3ea1, entries=150, sequenceid=156, filesize=11.9 K 2024-11-14T02:43:54,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/4fcc4358ef834ff2935e99ac5be9b43c as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/4fcc4358ef834ff2935e99ac5be9b43c 2024-11-14T02:43:54,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/4fcc4358ef834ff2935e99ac5be9b43c, entries=150, sequenceid=156, filesize=11.9 K 2024-11-14T02:43:54,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 30ff5d65ef696bee83471a2834e374a0 in 1276ms, sequenceid=156, compaction requested=true 2024-11-14T02:43:54,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:43:54,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30ff5d65ef696bee83471a2834e374a0:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:43:54,535 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-14T02:43:54,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:54,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30ff5d65ef696bee83471a2834e374a0:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:43:54,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:54,535 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-14T02:43:54,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30ff5d65ef696bee83471a2834e374a0:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:43:54,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:54,538 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 203371 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-14T02:43:54,538 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 30ff5d65ef696bee83471a2834e374a0/A is initiating minor compaction (all files) 2024-11-14T02:43:54,538 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30ff5d65ef696bee83471a2834e374a0/A in TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:54,539 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/1fa24a5d86504c85b8feb961ace0ee4d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/0a11afe80ff64a3086639ac7b3d07101, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/f2462c25f9314afb8b8151fad5265132, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/dca56e5935e94b749d15f528727fd627, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/52c1bfba06c547398b0d663070eeb0ca, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/551a9a4ae4ec4dfc82539592d62f0532] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp, totalSize=198.6 K 2024-11-14T02:43:54,539 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:54,539 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. files: [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/1fa24a5d86504c85b8feb961ace0ee4d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/0a11afe80ff64a3086639ac7b3d07101, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/f2462c25f9314afb8b8151fad5265132, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/dca56e5935e94b749d15f528727fd627, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/52c1bfba06c547398b0d663070eeb0ca, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/551a9a4ae4ec4dfc82539592d62f0532] 2024-11-14T02:43:54,539 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72409 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-14T02:43:54,539 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 30ff5d65ef696bee83471a2834e374a0/B is initiating minor compaction (all files) 2024-11-14T02:43:54,539 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30ff5d65ef696bee83471a2834e374a0/B in TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:54,539 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/0d294997d59e4f1b85abb2304e47c74e, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/99b57bf902834728b3a0390c1276b68d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/ab1e6843dca245908288860aa03ed29e, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/80628c31304d46baa19157487156ab75, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/b0988fe5cc704184b40d2084937e8e28, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/35e8c183292741ffb5411427d08a3ea1] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp, totalSize=70.7 K 2024-11-14T02:43:54,540 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d294997d59e4f1b85abb2304e47c74e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731552226212 2024-11-14T02:43:54,540 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fa24a5d86504c85b8feb961ace0ee4d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731552226212 2024-11-14T02:43:54,540 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 99b57bf902834728b3a0390c1276b68d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731552226421 2024-11-14T02:43:54,541 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a11afe80ff64a3086639ac7b3d07101, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731552226421 2024-11-14T02:43:54,541 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting ab1e6843dca245908288860aa03ed29e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731552226559 2024-11-14T02:43:54,541 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2462c25f9314afb8b8151fad5265132, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731552226559 2024-11-14T02:43:54,541 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 80628c31304d46baa19157487156ab75, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731552227735 2024-11-14T02:43:54,542 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting b0988fe5cc704184b40d2084937e8e28, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731552229904 2024-11-14T02:43:54,542 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting dca56e5935e94b749d15f528727fd627, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731552227735 2024-11-14T02:43:54,542 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 35e8c183292741ffb5411427d08a3ea1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731552232115 2024-11-14T02:43:54,542 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52c1bfba06c547398b0d663070eeb0ca, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731552229904 2024-11-14T02:43:54,544 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 551a9a4ae4ec4dfc82539592d62f0532, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731552232114 2024-11-14T02:43:54,557 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:54,558 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30ff5d65ef696bee83471a2834e374a0#B#compaction#140 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:54,559 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/926444e146b6498daa03e8ba39e5d17b is 50, key is test_row_0/B:col10/1731552232115/Put/seqid=0 2024-11-14T02:43:54,564 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114e956cb731e5941028e149a429d632137_30ff5d65ef696bee83471a2834e374a0 store=[table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:54,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T02:43:54,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:54,569 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 30ff5d65ef696bee83471a2834e374a0 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T02:43:54,570 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114e956cb731e5941028e149a429d632137_30ff5d65ef696bee83471a2834e374a0, store=[table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:54,570 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e956cb731e5941028e149a429d632137_30ff5d65ef696bee83471a2834e374a0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T02:43:54,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d958ee1c9b604d1e89a8fff7b3b46a30_30ff5d65ef696bee83471a2834e374a0 is 50, key is test_row_0/A:col10/1731552233273/Put/seqid=0 2024-11-14T02:43:54,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741989_1165 (size=12459) 2024-11-14T02:43:54,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741990_1166 (size=4469) 2024-11-14T02:43:54,595 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30ff5d65ef696bee83471a2834e374a0#A#compaction#141 average throughput is 0.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:54,596 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/cba105f65c924f29a283b0a20f35caa0 is 175, key is test_row_0/A:col10/1731552232115/Put/seqid=0 2024-11-14T02:43:54,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741991_1167 (size=12304) 2024-11-14T02:43:54,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:54,611 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d958ee1c9b604d1e89a8fff7b3b46a30_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d958ee1c9b604d1e89a8fff7b3b46a30_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:54,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/3479877f754348c28f988dbc241b7d62, store: [table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:54,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/3479877f754348c28f988dbc241b7d62 is 175, key is test_row_0/A:col10/1731552233273/Put/seqid=0 2024-11-14T02:43:54,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741992_1168 (size=31413) 2024-11-14T02:43:54,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741993_1169 (size=31105) 2024-11-14T02:43:54,628 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/3479877f754348c28f988dbc241b7d62 2024-11-14T02:43:54,635 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/cba105f65c924f29a283b0a20f35caa0 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/cba105f65c924f29a283b0a20f35caa0 2024-11-14T02:43:54,643 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 30ff5d65ef696bee83471a2834e374a0/A of 30ff5d65ef696bee83471a2834e374a0 into cba105f65c924f29a283b0a20f35caa0(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:54,643 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:43:54,643 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., storeName=30ff5d65ef696bee83471a2834e374a0/A, priority=10, startTime=1731552234534; duration=0sec 2024-11-14T02:43:54,643 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:54,643 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30ff5d65ef696bee83471a2834e374a0:A 2024-11-14T02:43:54,644 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-14T02:43:54,648 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72409 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-14T02:43:54,649 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 30ff5d65ef696bee83471a2834e374a0/C is initiating minor compaction (all files) 2024-11-14T02:43:54,649 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30ff5d65ef696bee83471a2834e374a0/C in TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:54,649 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/1bfc5541fe794f13b4fcec03eee8d40a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/eb84132149704060ad64d038d599fec1, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/1fde2989bfb443d09fced3c3552abcc6, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/e95d4817c038457dadf2a7620911821c, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/f952d17a299c4c78b13a08c7e437127c, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/4fcc4358ef834ff2935e99ac5be9b43c] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp, totalSize=70.7 K 2024-11-14T02:43:54,649 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bfc5541fe794f13b4fcec03eee8d40a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731552226212 2024-11-14T02:43:54,650 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb84132149704060ad64d038d599fec1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731552226421 2024-11-14T02:43:54,650 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fde2989bfb443d09fced3c3552abcc6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731552226559 2024-11-14T02:43:54,650 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting e95d4817c038457dadf2a7620911821c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731552227735 2024-11-14T02:43:54,651 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting f952d17a299c4c78b13a08c7e437127c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731552229904 2024-11-14T02:43:54,651 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fcc4358ef834ff2935e99ac5be9b43c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731552232115 2024-11-14T02:43:54,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/e240367aec0047989d7b0cdd9cd85d78 is 50, key is test_row_0/B:col10/1731552233273/Put/seqid=0 2024-11-14T02:43:54,674 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30ff5d65ef696bee83471a2834e374a0#C#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:54,674 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/a9183dca9c144216973bb7e0354c0157 is 50, key is test_row_0/C:col10/1731552232115/Put/seqid=0 2024-11-14T02:43:54,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741994_1170 (size=12151) 2024-11-14T02:43:54,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741995_1171 (size=12459) 2024-11-14T02:43:55,002 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/926444e146b6498daa03e8ba39e5d17b as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/926444e146b6498daa03e8ba39e5d17b 2024-11-14T02:43:55,010 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 30ff5d65ef696bee83471a2834e374a0/B of 30ff5d65ef696bee83471a2834e374a0 into 926444e146b6498daa03e8ba39e5d17b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:55,010 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:43:55,010 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., storeName=30ff5d65ef696bee83471a2834e374a0/B, priority=10, startTime=1731552234535; duration=0sec 2024-11-14T02:43:55,010 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:55,010 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30ff5d65ef696bee83471a2834e374a0:B 2024-11-14T02:43:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T02:43:55,096 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/e240367aec0047989d7b0cdd9cd85d78 2024-11-14T02:43:55,112 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/a9183dca9c144216973bb7e0354c0157 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/a9183dca9c144216973bb7e0354c0157 2024-11-14T02:43:55,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/e7e1dd5f04b24f67bfb159108929ae38 is 50, key is test_row_0/C:col10/1731552233273/Put/seqid=0 2024-11-14T02:43:55,120 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 30ff5d65ef696bee83471a2834e374a0/C of 30ff5d65ef696bee83471a2834e374a0 into a9183dca9c144216973bb7e0354c0157(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:55,120 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:43:55,120 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., storeName=30ff5d65ef696bee83471a2834e374a0/C, priority=10, startTime=1731552234535; duration=0sec 2024-11-14T02:43:55,121 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:55,121 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30ff5d65ef696bee83471a2834e374a0:C 2024-11-14T02:43:55,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741996_1172 (size=12151) 2024-11-14T02:43:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:55,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:43:55,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552295452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:55,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552295453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:55,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:55,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:55,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552295453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:55,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552295456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:55,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552295456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:55,529 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/e7e1dd5f04b24f67bfb159108929ae38 2024-11-14T02:43:55,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/3479877f754348c28f988dbc241b7d62 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/3479877f754348c28f988dbc241b7d62 2024-11-14T02:43:55,540 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/3479877f754348c28f988dbc241b7d62, entries=150, sequenceid=168, filesize=30.4 K 2024-11-14T02:43:55,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/e240367aec0047989d7b0cdd9cd85d78 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/e240367aec0047989d7b0cdd9cd85d78 2024-11-14T02:43:55,545 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/e240367aec0047989d7b0cdd9cd85d78, entries=150, sequenceid=168, filesize=11.9 K 2024-11-14T02:43:55,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/e7e1dd5f04b24f67bfb159108929ae38 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/e7e1dd5f04b24f67bfb159108929ae38 2024-11-14T02:43:55,551 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/e7e1dd5f04b24f67bfb159108929ae38, entries=150, sequenceid=168, filesize=11.9 K 2024-11-14T02:43:55,552 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 30ff5d65ef696bee83471a2834e374a0 in 983ms, sequenceid=168, compaction requested=false 2024-11-14T02:43:55,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:43:55,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:55,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-14T02:43:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-14T02:43:55,555 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-14T02:43:55,555 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5980 sec 2024-11-14T02:43:55,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.6020 sec 2024-11-14T02:43:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:55,566 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:43:55,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30ff5d65ef696bee83471a2834e374a0 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-14T02:43:55,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e5ab84ff370541558dbc7b263fee89a4_30ff5d65ef696bee83471a2834e374a0 is 50, key is test_row_0/A:col10/1731552235565/Put/seqid=0 2024-11-14T02:43:55,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552295569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:55,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:55,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552295570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552295569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:55,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:55,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552295571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:55,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:55,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:55,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552295577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:55,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:55,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:55,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:55,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:55,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741997_1173 (size=12304) 2024-11-14T02:43:55,582 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:55,587 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e5ab84ff370541558dbc7b263fee89a4_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e5ab84ff370541558dbc7b263fee89a4_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:55,588 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/9eeca47c76454916af4168e1ce28c9e1, store: [table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:55,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/9eeca47c76454916af4168e1ce28c9e1 is 175, key is test_row_0/A:col10/1731552235565/Put/seqid=0 2024-11-14T02:43:55,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741998_1174 (size=31105) 2024-11-14T02:43:55,596 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/9eeca47c76454916af4168e1ce28c9e1 2024-11-14T02:43:55,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/7b31254c0ade461f85ef4f79113d3235 is 50, key is test_row_0/B:col10/1731552235565/Put/seqid=0 2024-11-14T02:43:55,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741999_1175 (size=12151) 2024-11-14T02:43:55,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552295683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:55,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552295683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:55,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552295684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552295684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552295684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:55,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:55,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552295894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:55,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:55,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552295894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:55,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:55,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552295894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:55,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:55,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552295894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:55,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:55,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552295893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:55,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:55,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:56,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/7b31254c0ade461f85ef4f79113d3235 2024-11-14T02:43:56,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/7b7e6a25b3fd46fbbee3db5451710351 is 50, key is test_row_0/C:col10/1731552235565/Put/seqid=0 2024-11-14T02:43:56,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742000_1176 (size=12151) 2024-11-14T02:43:56,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T02:43:56,104 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:56,104 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:43:56,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-14T02:43:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T02:43:56,107 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:43:56,108 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:43:56,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:43:56,123 INFO [master/83c5a5c119d5:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-14T02:43:56,123 INFO [master/83c5a5c119d5:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-14T02:43:56,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:56,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552296203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:56,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:56,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552296203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:56,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552296204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552296203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552296204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T02:43:56,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T02:43:56,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:56,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:43:56,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:56,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:56,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:56,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T02:43:56,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:56,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:43:56,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:56,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:56,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:56,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:56,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T02:43:56,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/7b7e6a25b3fd46fbbee3db5451710351 2024-11-14T02:43:56,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/9eeca47c76454916af4168e1ce28c9e1 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/9eeca47c76454916af4168e1ce28c9e1 2024-11-14T02:43:56,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/9eeca47c76454916af4168e1ce28c9e1, entries=150, sequenceid=199, filesize=30.4 K 2024-11-14T02:43:56,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/7b31254c0ade461f85ef4f79113d3235 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/7b31254c0ade461f85ef4f79113d3235 2024-11-14T02:43:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/7b31254c0ade461f85ef4f79113d3235, entries=150, sequenceid=199, filesize=11.9 K 2024-11-14T02:43:56,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/7b7e6a25b3fd46fbbee3db5451710351 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/7b7e6a25b3fd46fbbee3db5451710351 2024-11-14T02:43:56,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/7b7e6a25b3fd46fbbee3db5451710351, entries=150, sequenceid=199, filesize=11.9 K 2024-11-14T02:43:56,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 30ff5d65ef696bee83471a2834e374a0 in 911ms, sequenceid=199, compaction requested=true 2024-11-14T02:43:56,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:43:56,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30ff5d65ef696bee83471a2834e374a0:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:43:56,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:56,478 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:56,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30ff5d65ef696bee83471a2834e374a0:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:43:56,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:56,478 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:56,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30ff5d65ef696bee83471a2834e374a0:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:43:56,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:56,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,479 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:56,479 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 30ff5d65ef696bee83471a2834e374a0/A is initiating minor compaction (all files) 2024-11-14T02:43:56,479 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30ff5d65ef696bee83471a2834e374a0/A in TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:56,479 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/cba105f65c924f29a283b0a20f35caa0, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/3479877f754348c28f988dbc241b7d62, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/9eeca47c76454916af4168e1ce28c9e1] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp, totalSize=91.4 K 2024-11-14T02:43:56,479 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:56,480 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. files: [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/cba105f65c924f29a283b0a20f35caa0, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/3479877f754348c28f988dbc241b7d62, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/9eeca47c76454916af4168e1ce28c9e1] 2024-11-14T02:43:56,480 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:56,480 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 30ff5d65ef696bee83471a2834e374a0/B is initiating minor compaction (all files) 2024-11-14T02:43:56,480 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30ff5d65ef696bee83471a2834e374a0/B in TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:56,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,481 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/926444e146b6498daa03e8ba39e5d17b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/e240367aec0047989d7b0cdd9cd85d78, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/7b31254c0ade461f85ef4f79113d3235] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp, totalSize=35.9 K 2024-11-14T02:43:56,481 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting cba105f65c924f29a283b0a20f35caa0, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731552232115 2024-11-14T02:43:56,481 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 926444e146b6498daa03e8ba39e5d17b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731552232115 2024-11-14T02:43:56,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,482 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3479877f754348c28f988dbc241b7d62, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731552233273 2024-11-14T02:43:56,482 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting e240367aec0047989d7b0cdd9cd85d78, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731552233273 2024-11-14T02:43:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,484 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9eeca47c76454916af4168e1ce28c9e1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731552235565 2024-11-14T02:43:56,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,484 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b31254c0ade461f85ef4f79113d3235, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731552235565 2024-11-14T02:43:56,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,496 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:56,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,502 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30ff5d65ef696bee83471a2834e374a0#B#compaction#150 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,503 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/c893084ce02f466f85f6c688c3868e29 is 50, key is test_row_0/B:col10/1731552235565/Put/seqid=0 2024-11-14T02:43:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,512 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111472c0a72b32a5455cbc26048ec6bc8e2f_30ff5d65ef696bee83471a2834e374a0 store=[table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,519 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111472c0a72b32a5455cbc26048ec6bc8e2f_30ff5d65ef696bee83471a2834e374a0, store=[table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:56,519 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111472c0a72b32a5455cbc26048ec6bc8e2f_30ff5d65ef696bee83471a2834e374a0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742001_1177 (size=12561) 2024-11-14T02:43:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742002_1178 (size=4469) 2024-11-14T02:43:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T02:43:56,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:56,567 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30ff5d65ef696bee83471a2834e374a0#A#compaction#149 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:56,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,567 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 30ff5d65ef696bee83471a2834e374a0 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-14T02:43:56,568 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/269e001169a4419989145f5a9320e40f is 175, key is test_row_0/A:col10/1731552235565/Put/seqid=0 2024-11-14T02:43:56,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ff7a6cdb041a45f2ac0238d2e6357a0f_30ff5d65ef696bee83471a2834e374a0 is 50, key is test_row_0/A:col10/1731552235575/Put/seqid=0 2024-11-14T02:43:56,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742003_1179 (size=31515) 2024-11-14T02:43:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,600 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/269e001169a4419989145f5a9320e40f as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/269e001169a4419989145f5a9320e40f 2024-11-14T02:43:56,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,607 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30ff5d65ef696bee83471a2834e374a0/A of 30ff5d65ef696bee83471a2834e374a0 into 269e001169a4419989145f5a9320e40f(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:56,607 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:43:56,608 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., storeName=30ff5d65ef696bee83471a2834e374a0/A, priority=13, startTime=1731552236477; duration=0sec 2024-11-14T02:43:56,608 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,608 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30ff5d65ef696bee83471a2834e374a0:A 2024-11-14T02:43:56,608 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:56,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,610 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:56,610 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 30ff5d65ef696bee83471a2834e374a0/C is initiating minor compaction (all files) 2024-11-14T02:43:56,610 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30ff5d65ef696bee83471a2834e374a0/C in TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:56,610 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/a9183dca9c144216973bb7e0354c0157, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/e7e1dd5f04b24f67bfb159108929ae38, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/7b7e6a25b3fd46fbbee3db5451710351] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp, totalSize=35.9 K 2024-11-14T02:43:56,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,610 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9183dca9c144216973bb7e0354c0157, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731552232115 2024-11-14T02:43:56,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,611 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7e1dd5f04b24f67bfb159108929ae38, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731552233273 2024-11-14T02:43:56,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,611 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b7e6a25b3fd46fbbee3db5451710351, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731552235565 2024-11-14T02:43:56,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742004_1180 (size=9814) 2024-11-14T02:43:56,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,646 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30ff5d65ef696bee83471a2834e374a0#C#compaction#152 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,646 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/0a7bfc92b41c43549fb39ef3edc0033e is 50, key is test_row_0/C:col10/1731552235565/Put/seqid=0 2024-11-14T02:43:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742005_1181 (size=12561) 2024-11-14T02:43:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:56,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:43:56,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T02:43:56,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:56,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:56,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552296766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:56,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:56,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:56,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552296767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:56,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:56,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:56,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552296776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:56,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:56,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552296777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:56,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:56,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552296777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:56,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:56,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:56,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:56,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:56,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552296883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:56,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:56,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:56,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552296883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:56,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:56,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552296883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:56,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:56,886 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,886 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,886 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:56,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:56,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552296907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:56,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:56,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552296908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:56,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:56,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:56,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:56,946 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/c893084ce02f466f85f6c688c3868e29 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/c893084ce02f466f85f6c688c3868e29 2024-11-14T02:43:56,952 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30ff5d65ef696bee83471a2834e374a0/B of 30ff5d65ef696bee83471a2834e374a0 into c893084ce02f466f85f6c688c3868e29(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:56,952 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:43:56,952 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., storeName=30ff5d65ef696bee83471a2834e374a0/B, priority=13, startTime=1731552236478; duration=0sec 2024-11-14T02:43:56,952 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:56,952 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30ff5d65ef696bee83471a2834e374a0:B 2024-11-14T02:43:57,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:57,044 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ff7a6cdb041a45f2ac0238d2e6357a0f_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ff7a6cdb041a45f2ac0238d2e6357a0f_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:57,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/bf69996fef5244e4994406f4b6f36433, store: [table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:57,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/bf69996fef5244e4994406f4b6f36433 is 175, key is test_row_0/A:col10/1731552235575/Put/seqid=0 2024-11-14T02:43:57,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742006_1182 (size=22461) 2024-11-14T02:43:57,087 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/0a7bfc92b41c43549fb39ef3edc0033e as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/0a7bfc92b41c43549fb39ef3edc0033e 2024-11-14T02:43:57,092 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30ff5d65ef696bee83471a2834e374a0/C of 30ff5d65ef696bee83471a2834e374a0 into 0a7bfc92b41c43549fb39ef3edc0033e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:57,092 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:43:57,093 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., storeName=30ff5d65ef696bee83471a2834e374a0/C, priority=13, startTime=1731552236478; duration=0sec 2024-11-14T02:43:57,093 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:57,093 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30ff5d65ef696bee83471a2834e374a0:C 2024-11-14T02:43:57,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552297093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:57,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552297093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:57,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:57,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552297093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:57,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:57,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:57,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:57,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552297114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:57,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:57,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552297114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:57,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:57,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T02:43:57,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552297403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:57,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:57,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552297403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:57,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:57,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:57,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552297403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:57,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:57,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:57,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552297423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:57,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:57,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552297423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:57,451 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/bf69996fef5244e4994406f4b6f36433 2024-11-14T02:43:57,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/b5335804eee041489968c7fe284b66cf is 50, key is test_row_0/B:col10/1731552235575/Put/seqid=0 2024-11-14T02:43:57,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742007_1183 (size=9757) 2024-11-14T02:43:57,864 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/b5335804eee041489968c7fe284b66cf 2024-11-14T02:43:57,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/cc978d3be2e345cd9a09f4c40a54b145 is 50, key is test_row_0/C:col10/1731552235575/Put/seqid=0 2024-11-14T02:43:57,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742008_1184 (size=9757) 2024-11-14T02:43:57,877 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/cc978d3be2e345cd9a09f4c40a54b145 2024-11-14T02:43:57,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/bf69996fef5244e4994406f4b6f36433 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/bf69996fef5244e4994406f4b6f36433 2024-11-14T02:43:57,886 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/bf69996fef5244e4994406f4b6f36433, entries=100, sequenceid=207, filesize=21.9 K 2024-11-14T02:43:57,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/b5335804eee041489968c7fe284b66cf as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/b5335804eee041489968c7fe284b66cf 2024-11-14T02:43:57,892 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/b5335804eee041489968c7fe284b66cf, entries=100, sequenceid=207, filesize=9.5 K 2024-11-14T02:43:57,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/cc978d3be2e345cd9a09f4c40a54b145 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/cc978d3be2e345cd9a09f4c40a54b145 2024-11-14T02:43:57,898 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/cc978d3be2e345cd9a09f4c40a54b145, entries=100, sequenceid=207, filesize=9.5 K 2024-11-14T02:43:57,899 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=174.43 KB/178620 for 30ff5d65ef696bee83471a2834e374a0 in 1332ms, sequenceid=207, compaction requested=false 2024-11-14T02:43:57,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:43:57,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:57,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-14T02:43:57,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-14T02:43:57,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-14T02:43:57,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7940 sec 2024-11-14T02:43:57,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.8000 sec 2024-11-14T02:43:57,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:57,916 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:43:57,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30ff5d65ef696bee83471a2834e374a0 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-14T02:43:57,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:57,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552297920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:57,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:57,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:57,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552297921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:57,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148a77d155cb82492b9107f8bfcc831930_30ff5d65ef696bee83471a2834e374a0 is 50, key is test_row_0/A:col10/1731552237915/Put/seqid=0 2024-11-14T02:43:57,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:57,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552297923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:57,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:57,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742009_1185 (size=14794) 2024-11-14T02:43:57,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:57,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552297933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:57,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:57,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:57,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552297933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:57,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:57,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:58,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:58,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552298033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:58,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:58,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552298033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:58,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:58,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552298244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:58,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:58,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552298244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:58,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:58,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:58,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:58,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T02:43:58,253 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:58,254 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:43:58,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-14T02:43:58,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T02:43:58,255 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:43:58,256 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:43:58,256 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:43:58,329 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,341 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148a77d155cb82492b9107f8bfcc831930_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148a77d155cb82492b9107f8bfcc831930_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:58,342 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/4edd48534875478c8a4c126a42dd13ec, store: [table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:58,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/4edd48534875478c8a4c126a42dd13ec is 175, key is test_row_0/A:col10/1731552237915/Put/seqid=0 2024-11-14T02:43:58,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742010_1186 (size=39749) 2024-11-14T02:43:58,348 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/4edd48534875478c8a4c126a42dd13ec 2024-11-14T02:43:58,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/ba4106b2dbf243f891e9ee6a26b3f3c4 is 50, key is test_row_0/B:col10/1731552237915/Put/seqid=0 2024-11-14T02:43:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T02:43:58,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742011_1187 (size=12151) 2024-11-14T02:43:58,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T02:43:58,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:58,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:43:58,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:58,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:58,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:58,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552298553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:58,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:58,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:58,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:58,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552298553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:58,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:58,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T02:43:58,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:58,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:43:58,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:58,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:58,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:58,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:58,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T02:43:58,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T02:43:58,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:58,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:43:58,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:58,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:58,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:58,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/ba4106b2dbf243f891e9ee6a26b3f3c4 2024-11-14T02:43:58,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/e770049053e04a41bec5b1145f3a6af6 is 50, key is test_row_0/C:col10/1731552237915/Put/seqid=0 2024-11-14T02:43:58,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742012_1188 (size=12151) 2024-11-14T02:43:58,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/e770049053e04a41bec5b1145f3a6af6 2024-11-14T02:43:58,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/4edd48534875478c8a4c126a42dd13ec as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/4edd48534875478c8a4c126a42dd13ec 2024-11-14T02:43:58,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/4edd48534875478c8a4c126a42dd13ec, entries=200, sequenceid=240, filesize=38.8 K 2024-11-14T02:43:58,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/ba4106b2dbf243f891e9ee6a26b3f3c4 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/ba4106b2dbf243f891e9ee6a26b3f3c4 2024-11-14T02:43:58,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/ba4106b2dbf243f891e9ee6a26b3f3c4, entries=150, sequenceid=240, filesize=11.9 K 2024-11-14T02:43:58,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/e770049053e04a41bec5b1145f3a6af6 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/e770049053e04a41bec5b1145f3a6af6 2024-11-14T02:43:58,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/e770049053e04a41bec5b1145f3a6af6, entries=150, sequenceid=240, filesize=11.9 K 2024-11-14T02:43:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 30ff5d65ef696bee83471a2834e374a0 in 890ms, sequenceid=240, compaction requested=true 2024-11-14T02:43:58,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:43:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30ff5d65ef696bee83471a2834e374a0:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:43:58,806 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:58,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:58,807 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30ff5d65ef696bee83471a2834e374a0:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:43:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:58,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30ff5d65ef696bee83471a2834e374a0:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:43:58,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,808 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93725 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,808 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 30ff5d65ef696bee83471a2834e374a0/A is initiating minor compaction (all files) 2024-11-14T02:43:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,808 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30ff5d65ef696bee83471a2834e374a0/A in TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:58,808 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/269e001169a4419989145f5a9320e40f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/bf69996fef5244e4994406f4b6f36433, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/4edd48534875478c8a4c126a42dd13ec] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp, totalSize=91.5 K 2024-11-14T02:43:58,808 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:58,808 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. files: [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/269e001169a4419989145f5a9320e40f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/bf69996fef5244e4994406f4b6f36433, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/4edd48534875478c8a4c126a42dd13ec] 2024-11-14T02:43:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,808 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:58,808 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 30ff5d65ef696bee83471a2834e374a0/B is initiating minor compaction (all files) 2024-11-14T02:43:58,808 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30ff5d65ef696bee83471a2834e374a0/B in TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:58,809 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/c893084ce02f466f85f6c688c3868e29, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/b5335804eee041489968c7fe284b66cf, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/ba4106b2dbf243f891e9ee6a26b3f3c4] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp, totalSize=33.7 K 2024-11-14T02:43:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,809 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 269e001169a4419989145f5a9320e40f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731552235565 2024-11-14T02:43:58,809 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting c893084ce02f466f85f6c688c3868e29, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731552235565 2024-11-14T02:43:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,810 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf69996fef5244e4994406f4b6f36433, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731552235568 2024-11-14T02:43:58,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,810 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting b5335804eee041489968c7fe284b66cf, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731552235568 2024-11-14T02:43:58,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,810 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting ba4106b2dbf243f891e9ee6a26b3f3c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731552236759 2024-11-14T02:43:58,810 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4edd48534875478c8a4c126a42dd13ec, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731552236754 2024-11-14T02:43:58,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,818 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:58,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,832 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114d6c819cb72b94568a3df729b5404eb1d_30ff5d65ef696bee83471a2834e374a0 store=[table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:58,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,833 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30ff5d65ef696bee83471a2834e374a0#B#compaction#159 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,834 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/3e4933ab52624073a7b87a955ceff00f is 50, key is test_row_0/B:col10/1731552237915/Put/seqid=0 2024-11-14T02:43:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,838 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114d6c819cb72b94568a3df729b5404eb1d_30ff5d65ef696bee83471a2834e374a0, store=[table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,838 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d6c819cb72b94568a3df729b5404eb1d_30ff5d65ef696bee83471a2834e374a0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742013_1189 (size=12663) 2024-11-14T02:43:58,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,846 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/3e4933ab52624073a7b87a955ceff00f as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/3e4933ab52624073a7b87a955ceff00f 2024-11-14T02:43:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,853 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30ff5d65ef696bee83471a2834e374a0/B of 30ff5d65ef696bee83471a2834e374a0 into 3e4933ab52624073a7b87a955ceff00f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:58,853 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:43:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,853 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., storeName=30ff5d65ef696bee83471a2834e374a0/B, priority=13, startTime=1731552238806; duration=0sec 2024-11-14T02:43:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,853 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:58,853 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30ff5d65ef696bee83471a2834e374a0:B 2024-11-14T02:43:58,854 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:58,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,855 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:58,855 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 30ff5d65ef696bee83471a2834e374a0/C is initiating minor compaction (all files) 2024-11-14T02:43:58,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,855 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30ff5d65ef696bee83471a2834e374a0/C in TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:58,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,855 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/0a7bfc92b41c43549fb39ef3edc0033e, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/cc978d3be2e345cd9a09f4c40a54b145, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/e770049053e04a41bec5b1145f3a6af6] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp, totalSize=33.7 K 2024-11-14T02:43:58,856 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a7bfc92b41c43549fb39ef3edc0033e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731552235565 2024-11-14T02:43:58,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,856 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting cc978d3be2e345cd9a09f4c40a54b145, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731552235568 2024-11-14T02:43:58,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,856 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting e770049053e04a41bec5b1145f3a6af6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731552236759 2024-11-14T02:43:58,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,865 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30ff5d65ef696bee83471a2834e374a0#C#compaction#160 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:58,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,866 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/9094835098ac4df5a093bb3aa35cc548 is 50, key is test_row_0/C:col10/1731552237915/Put/seqid=0 2024-11-14T02:43:58,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T02:43:58,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:58,868 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 30ff5d65ef696bee83471a2834e374a0 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-14T02:43:58,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742015_1191 (size=12663) 2024-11-14T02:43:58,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d2db24f18f2d46d0af8b9946e002989d_30ff5d65ef696bee83471a2834e374a0 is 50, key is test_row_2/A:col10/1731552237919/Put/seqid=0 2024-11-14T02:43:58,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T02:43:58,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742014_1190 (size=4469) 2024-11-14T02:43:58,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,891 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30ff5d65ef696bee83471a2834e374a0#A#compaction#158 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:58,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,891 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/4abf3314e1584f7aaff26f99d107d763 is 175, key is test_row_0/A:col10/1731552237915/Put/seqid=0 2024-11-14T02:43:58,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742016_1192 (size=7324) 2024-11-14T02:43:58,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742017_1193 (size=31617) 2024-11-14T02:43:58,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,928 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/4abf3314e1584f7aaff26f99d107d763 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/4abf3314e1584f7aaff26f99d107d763 2024-11-14T02:43:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,935 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30ff5d65ef696bee83471a2834e374a0/A of 30ff5d65ef696bee83471a2834e374a0 into 4abf3314e1584f7aaff26f99d107d763(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:58,935 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:43:58,935 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., storeName=30ff5d65ef696bee83471a2834e374a0/A, priority=13, startTime=1731552238806; duration=0sec 2024-11-14T02:43:58,935 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:58,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,935 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30ff5d65ef696bee83471a2834e374a0:A 2024-11-14T02:43:58,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:58,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:43:58,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:58,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:59,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552299071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:59,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:59,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:59,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552299072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:59,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:59,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:59,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552299072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:59,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:59,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552299076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:59,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:59,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552299078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:59,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:59,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:59,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:59,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:59,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552299183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:59,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:59,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552299183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:59,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:59,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:59,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:59,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552299183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:59,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:59,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:59,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552299184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:59,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:59,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:59,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552299193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:59,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:59,283 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/9094835098ac4df5a093bb3aa35cc548 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/9094835098ac4df5a093bb3aa35cc548 2024-11-14T02:43:59,288 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30ff5d65ef696bee83471a2834e374a0/C of 30ff5d65ef696bee83471a2834e374a0 into 9094835098ac4df5a093bb3aa35cc548(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:59,288 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:43:59,289 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., storeName=30ff5d65ef696bee83471a2834e374a0/C, priority=13, startTime=1731552238807; duration=0sec 2024-11-14T02:43:59,289 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:59,289 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30ff5d65ef696bee83471a2834e374a0:C 2024-11-14T02:43:59,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:59,307 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d2db24f18f2d46d0af8b9946e002989d_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d2db24f18f2d46d0af8b9946e002989d_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:43:59,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/8c93ab4ae1ba4a9b8e986d0a6e395d17, store: [table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:43:59,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/8c93ab4ae1ba4a9b8e986d0a6e395d17 is 175, key is test_row_2/A:col10/1731552237919/Put/seqid=0 2024-11-14T02:43:59,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742018_1194 (size=13815) 2024-11-14T02:43:59,319 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/8c93ab4ae1ba4a9b8e986d0a6e395d17 2024-11-14T02:43:59,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/a20ebd32a3ea43aa9848cbc3593bc662 is 50, key is test_row_2/B:col10/1731552237919/Put/seqid=0 2024-11-14T02:43:59,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742019_1195 (size=7365) 2024-11-14T02:43:59,338 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/a20ebd32a3ea43aa9848cbc3593bc662 2024-11-14T02:43:59,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/495cc90bb3fe4eac8bceb675dc022067 is 50, key is test_row_2/C:col10/1731552237919/Put/seqid=0 2024-11-14T02:43:59,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742020_1196 (size=7365) 2024-11-14T02:43:59,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T02:43:59,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:59,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552299393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552299394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:59,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:59,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:59,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552299394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:59,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:59,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552299394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:59,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:59,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:59,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:59,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552299403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:59,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:59,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:59,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552299703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:59,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:59,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552299703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:59,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:59,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:59,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:59,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552299704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:59,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:59,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552299704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:59,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:59,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:59,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:43:59,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552299713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:43:59,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:59,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:43:59,752 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/495cc90bb3fe4eac8bceb675dc022067 2024-11-14T02:43:59,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/8c93ab4ae1ba4a9b8e986d0a6e395d17 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/8c93ab4ae1ba4a9b8e986d0a6e395d17 2024-11-14T02:43:59,762 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/8c93ab4ae1ba4a9b8e986d0a6e395d17, entries=50, sequenceid=247, filesize=13.5 K 2024-11-14T02:43:59,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/a20ebd32a3ea43aa9848cbc3593bc662 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/a20ebd32a3ea43aa9848cbc3593bc662 2024-11-14T02:43:59,768 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/a20ebd32a3ea43aa9848cbc3593bc662, entries=50, sequenceid=247, filesize=7.2 K 2024-11-14T02:43:59,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/495cc90bb3fe4eac8bceb675dc022067 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/495cc90bb3fe4eac8bceb675dc022067 2024-11-14T02:43:59,776 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/495cc90bb3fe4eac8bceb675dc022067, entries=50, sequenceid=247, filesize=7.2 K 2024-11-14T02:43:59,777 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=181.14 KB/185490 for 30ff5d65ef696bee83471a2834e374a0 in 909ms, sequenceid=247, compaction requested=false 2024-11-14T02:43:59,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:43:59,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:43:59,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-14T02:43:59,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-14T02:43:59,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-14T02:43:59,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5220 sec 2024-11-14T02:43:59,782 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.5270 sec 2024-11-14T02:44:00,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:00,215 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:44:00,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30ff5d65ef696bee83471a2834e374a0 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-14T02:44:00,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:00,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552300213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:00,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:00,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:00,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:00,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:00,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552300216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:00,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:00,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:00,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:00,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:00,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552300216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:00,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552300217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:00,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:00,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552300223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:00,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:00,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:00,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:00,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a944e65338ed416fb211b80fed9ac149_30ff5d65ef696bee83471a2834e374a0 is 50, key is test_row_0/A:col10/1731552240215/Put/seqid=0 2024-11-14T02:44:00,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742021_1197 (size=20074) 2024-11-14T02:44:00,234 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,238 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a944e65338ed416fb211b80fed9ac149_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a944e65338ed416fb211b80fed9ac149_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:00,240 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/fbd5da81db5946e9a8b4aa4674e215e0, store: [table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:44:00,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/fbd5da81db5946e9a8b4aa4674e215e0 is 175, key is test_row_0/A:col10/1731552240215/Put/seqid=0 2024-11-14T02:44:00,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742022_1198 (size=57333) 2024-11-14T02:44:00,259 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=67.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/fbd5da81db5946e9a8b4aa4674e215e0 2024-11-14T02:44:00,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/2929cfb0d0df4c48adec27478eb9661a is 50, key is test_row_0/B:col10/1731552240215/Put/seqid=0 2024-11-14T02:44:00,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742023_1199 (size=12301) 2024-11-14T02:44:00,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/2929cfb0d0df4c48adec27478eb9661a 2024-11-14T02:44:00,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/21bb355770664c91a4e8c34bf054a961 is 50, key is test_row_0/C:col10/1731552240215/Put/seqid=0 2024-11-14T02:44:00,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742024_1200 (size=12301) 2024-11-14T02:44:00,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552300323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:00,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:00,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:00,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:00,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:00,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552300324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:00,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:00,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:00,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:00,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:00,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552300324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:00,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:00,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:00,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:00,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T02:44:00,404 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:00,404 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:00,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-14T02:44:00,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T02:44:00,406 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:00,406 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:00,406 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T02:44:00,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552300533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:00,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:00,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:00,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552300534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:00,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:00,534 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:00,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:00,534 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:00,534 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552300534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:00,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T02:44:00,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:00,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:44:00,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:00,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:00,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:00,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/21bb355770664c91a4e8c34bf054a961 2024-11-14T02:44:00,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/fbd5da81db5946e9a8b4aa4674e215e0 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/fbd5da81db5946e9a8b4aa4674e215e0 2024-11-14T02:44:00,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/fbd5da81db5946e9a8b4aa4674e215e0, entries=300, sequenceid=282, filesize=56.0 K 2024-11-14T02:44:00,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T02:44:00,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/2929cfb0d0df4c48adec27478eb9661a as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/2929cfb0d0df4c48adec27478eb9661a 2024-11-14T02:44:00,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:00,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:44:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:00,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:00,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:00,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/2929cfb0d0df4c48adec27478eb9661a, entries=150, sequenceid=282, filesize=12.0 K 2024-11-14T02:44:00,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/21bb355770664c91a4e8c34bf054a961 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/21bb355770664c91a4e8c34bf054a961 2024-11-14T02:44:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/21bb355770664c91a4e8c34bf054a961, entries=150, sequenceid=282, filesize=12.0 K 2024-11-14T02:44:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T02:44:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~201.27 KB/206100, heapSize ~528.05 KB/540720, currentSize=0 B/0 for 30ff5d65ef696bee83471a2834e374a0 in 509ms, sequenceid=282, compaction requested=true 2024-11-14T02:44:00,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:44:00,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30ff5d65ef696bee83471a2834e374a0:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:44:00,724 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:00,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:00,724 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30ff5d65ef696bee83471a2834e374a0:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:44:00,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:00,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30ff5d65ef696bee83471a2834e374a0:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:44:00,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,728 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102765 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:00,729 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 30ff5d65ef696bee83471a2834e374a0/A is initiating minor compaction (all files) 2024-11-14T02:44:00,729 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30ff5d65ef696bee83471a2834e374a0/A in TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:00,729 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/4abf3314e1584f7aaff26f99d107d763, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/8c93ab4ae1ba4a9b8e986d0a6e395d17, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/fbd5da81db5946e9a8b4aa4674e215e0] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp, totalSize=100.4 K 2024-11-14T02:44:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,729 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:00,729 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. files: [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/4abf3314e1584f7aaff26f99d107d763, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/8c93ab4ae1ba4a9b8e986d0a6e395d17, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/fbd5da81db5946e9a8b4aa4674e215e0] 2024-11-14T02:44:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,730 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32329 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:00,730 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 30ff5d65ef696bee83471a2834e374a0/B is initiating minor compaction (all files) 2024-11-14T02:44:00,730 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30ff5d65ef696bee83471a2834e374a0/B in TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:00,730 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/3e4933ab52624073a7b87a955ceff00f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/a20ebd32a3ea43aa9848cbc3593bc662, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/2929cfb0d0df4c48adec27478eb9661a] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp, totalSize=31.6 K 2024-11-14T02:44:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,731 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e4933ab52624073a7b87a955ceff00f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731552236759 2024-11-14T02:44:00,731 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4abf3314e1584f7aaff26f99d107d763, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731552236759 2024-11-14T02:44:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,731 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c93ab4ae1ba4a9b8e986d0a6e395d17, keycount=50, bloomtype=ROW, size=13.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731552237919 2024-11-14T02:44:00,731 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting a20ebd32a3ea43aa9848cbc3593bc662, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731552237919 2024-11-14T02:44:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,732 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 2929cfb0d0df4c48adec27478eb9661a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731552239070 2024-11-14T02:44:00,732 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbd5da81db5946e9a8b4aa4674e215e0, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731552239070 2024-11-14T02:44:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,744 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:44:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,751 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30ff5d65ef696bee83471a2834e374a0#B#compaction#168 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:00,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,752 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/5afa504038a4445c8afacffe193ecf91 is 50, key is test_row_0/B:col10/1731552240215/Put/seqid=0 2024-11-14T02:44:00,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,756 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114d354d45fd3fa47b2b3295fedc8127905_30ff5d65ef696bee83471a2834e374a0 store=[table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:44:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,761 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114d354d45fd3fa47b2b3295fedc8127905_30ff5d65ef696bee83471a2834e374a0, store=[table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:44:00,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,761 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d354d45fd3fa47b2b3295fedc8127905_30ff5d65ef696bee83471a2834e374a0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:44:00,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742025_1201 (size=12915) 2024-11-14T02:44:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742026_1202 (size=4469) 2024-11-14T02:44:00,787 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30ff5d65ef696bee83471a2834e374a0#A#compaction#167 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:00,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,787 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/99f166cdcf28459c8401eb55da46949a is 175, key is test_row_0/A:col10/1731552240215/Put/seqid=0 2024-11-14T02:44:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,788 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/5afa504038a4445c8afacffe193ecf91 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/5afa504038a4445c8afacffe193ecf91 2024-11-14T02:44:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,797 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30ff5d65ef696bee83471a2834e374a0/B of 30ff5d65ef696bee83471a2834e374a0 into 5afa504038a4445c8afacffe193ecf91(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:00,798 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:44:00,798 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., storeName=30ff5d65ef696bee83471a2834e374a0/B, priority=13, startTime=1731552240724; duration=0sec 2024-11-14T02:44:00,798 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:00,798 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30ff5d65ef696bee83471a2834e374a0:B 2024-11-14T02:44:00,798 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:00,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,800 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32329 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:00,800 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 30ff5d65ef696bee83471a2834e374a0/C is initiating minor compaction (all files) 2024-11-14T02:44:00,800 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30ff5d65ef696bee83471a2834e374a0/C in TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:00,800 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/9094835098ac4df5a093bb3aa35cc548, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/495cc90bb3fe4eac8bceb675dc022067, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/21bb355770664c91a4e8c34bf054a961] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp, totalSize=31.6 K 2024-11-14T02:44:00,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,801 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 9094835098ac4df5a093bb3aa35cc548, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731552236759 2024-11-14T02:44:00,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,802 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 495cc90bb3fe4eac8bceb675dc022067, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731552237919 2024-11-14T02:44:00,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,802 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 21bb355770664c91a4e8c34bf054a961, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731552239070 2024-11-14T02:44:00,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,813 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30ff5d65ef696bee83471a2834e374a0#C#compaction#169 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:00,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,813 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/3f457c989fc24539b398857872f61557 is 50, key is test_row_0/C:col10/1731552240215/Put/seqid=0 2024-11-14T02:44:00,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742028_1204 (size=12915) 2024-11-14T02:44:00,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742027_1203 (size=31869) 2024-11-14T02:44:00,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,826 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/3f457c989fc24539b398857872f61557 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/3f457c989fc24539b398857872f61557 2024-11-14T02:44:00,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,835 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30ff5d65ef696bee83471a2834e374a0/C of 30ff5d65ef696bee83471a2834e374a0 into 3f457c989fc24539b398857872f61557(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:00,835 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:44:00,835 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., storeName=30ff5d65ef696bee83471a2834e374a0/C, priority=13, startTime=1731552240725; duration=0sec 2024-11-14T02:44:00,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,835 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:00,835 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30ff5d65ef696bee83471a2834e374a0:C 2024-11-14T02:44:00,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,839 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] io.ByteBuffAllocator(384): Pool already reached its max capacity : 186 and no free buffers now. Consider increasing the value for 'hbase.server.allocator.max.buffer.count' ? 2024-11-14T02:44:00,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T02:44:00,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:00,865 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 30ff5d65ef696bee83471a2834e374a0 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-14T02:44:00,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146bb8f8935e834726b0603837647386cc_30ff5d65ef696bee83471a2834e374a0 is 50, key is test_row_1/A:col10/1731552240856/Put/seqid=0 2024-11-14T02:44:00,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742029_1205 (size=9914) 2024-11-14T02:44:00,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,899 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146bb8f8935e834726b0603837647386cc_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146bb8f8935e834726b0603837647386cc_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:00,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/1b68e6d8366147be9fb06dcd74ef037a, store: [table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:44:00,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/1b68e6d8366147be9fb06dcd74ef037a is 175, key is test_row_1/A:col10/1731552240856/Put/seqid=0 2024-11-14T02:44:00,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742030_1206 (size=22559) 2024-11-14T02:44:00,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:44:00,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:00,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:00,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:01,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:01,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:01,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:01,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:01,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:01,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:01,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:01,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:01,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:01,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:01,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:01,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:01,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T02:44:01,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552301037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:01,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:01,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552301039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:01,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:01,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:01,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552301040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:01,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:01,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:01,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552301144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:01,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:01,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:01,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552301144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:01,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:01,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552301153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:01,226 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/99f166cdcf28459c8401eb55da46949a as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/99f166cdcf28459c8401eb55da46949a 2024-11-14T02:44:01,231 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30ff5d65ef696bee83471a2834e374a0/A of 30ff5d65ef696bee83471a2834e374a0 into 99f166cdcf28459c8401eb55da46949a(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:01,231 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:44:01,231 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., storeName=30ff5d65ef696bee83471a2834e374a0/A, priority=13, startTime=1731552240724; duration=0sec 2024-11-14T02:44:01,231 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:01,232 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30ff5d65ef696bee83471a2834e374a0:A 2024-11-14T02:44:01,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552301234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:01,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552301234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:01,237 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_0' in 30ff5d65ef696bee83471a2834e374a0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:44:01,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:01,237 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_0' in 30ff5d65ef696bee83471a2834e374a0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:44:01,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:01,310 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/1b68e6d8366147be9fb06dcd74ef037a 2024-11-14T02:44:01,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/57eb4eb016fe42dba767370b2c29e6a1 is 50, key is test_row_1/B:col10/1731552240856/Put/seqid=0 2024-11-14T02:44:01,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742031_1207 (size=7415) 2024-11-14T02:44:01,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552301353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:01,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:01,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552301353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:01,356 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,356 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,356 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:01,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:01,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552301363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:01,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T02:44:01,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552301664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:01,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:01,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552301664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:01,666 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:01,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552301673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:01,725 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/57eb4eb016fe42dba767370b2c29e6a1 2024-11-14T02:44:01,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/eca274203799419299e14cd435c8434d is 50, key is test_row_1/C:col10/1731552240856/Put/seqid=0 2024-11-14T02:44:01,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742032_1208 (size=7415) 2024-11-14T02:44:01,751 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/eca274203799419299e14cd435c8434d 2024-11-14T02:44:01,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/1b68e6d8366147be9fb06dcd74ef037a as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/1b68e6d8366147be9fb06dcd74ef037a 2024-11-14T02:44:01,762 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/1b68e6d8366147be9fb06dcd74ef037a, entries=100, sequenceid=289, filesize=22.0 K 2024-11-14T02:44:01,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/57eb4eb016fe42dba767370b2c29e6a1 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/57eb4eb016fe42dba767370b2c29e6a1 2024-11-14T02:44:01,769 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/57eb4eb016fe42dba767370b2c29e6a1, entries=50, sequenceid=289, filesize=7.2 K 2024-11-14T02:44:01,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/eca274203799419299e14cd435c8434d as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/eca274203799419299e14cd435c8434d 2024-11-14T02:44:01,776 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/eca274203799419299e14cd435c8434d, entries=50, sequenceid=289, filesize=7.2 K 2024-11-14T02:44:01,777 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=187.85 KB/192360 for 30ff5d65ef696bee83471a2834e374a0 in 912ms, sequenceid=289, compaction requested=false 2024-11-14T02:44:01,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:44:01,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:01,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-14T02:44:01,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-14T02:44:01,780 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-14T02:44:01,780 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3720 sec 2024-11-14T02:44:01,782 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.3770 sec 2024-11-14T02:44:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:02,175 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:44:02,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30ff5d65ef696bee83471a2834e374a0 3/3 column families, dataSize=194.56 KB heapSize=510.52 KB 2024-11-14T02:44:02,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552302175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:02,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552302176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:02,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:02,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:02,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:02,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144a26c4a3389d487ca389b4331e35f75b_30ff5d65ef696bee83471a2834e374a0 is 50, key is test_row_0/A:col10/1731552241036/Put/seqid=0 2024-11-14T02:44:02,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:02,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552302183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:02,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:02,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:02,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:02,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742033_1209 (size=14994) 2024-11-14T02:44:02,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:02,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:02,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552302283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:02,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552302283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:02,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:02,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:02,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552302493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:02,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:02,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:02,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:02,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552302493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:02,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:02,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:02,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:02,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:02,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T02:44:02,554 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:02,554 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:02,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-14T02:44:02,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T02:44:02,556 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:02,557 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:02,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:02,596 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:02,600 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144a26c4a3389d487ca389b4331e35f75b_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144a26c4a3389d487ca389b4331e35f75b_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:02,601 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/a6a3da9ed11f465382ea365b9c941b5f, store: [table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:44:02,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/a6a3da9ed11f465382ea365b9c941b5f is 175, key is test_row_0/A:col10/1731552241036/Put/seqid=0 2024-11-14T02:44:02,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742034_1210 (size=39949) 2024-11-14T02:44:02,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T02:44:02,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-14T02:44:02,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:02,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:44:02,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:02,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:02,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:02,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:02,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552302803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:02,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:02,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:02,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:02,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552302803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:02,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:02,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:02,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:02,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:02,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-14T02:44:02,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:02,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:44:02,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:02,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:02,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:02,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T02:44:03,008 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=67.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/a6a3da9ed11f465382ea365b9c941b5f 2024-11-14T02:44:03,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-14T02:44:03,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:03,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:44:03,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:03,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:03,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:03,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/47644e66f834413298459a9124c65ea2 is 50, key is test_row_0/B:col10/1731552241036/Put/seqid=0 2024-11-14T02:44:03,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:03,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742035_1211 (size=12301) 2024-11-14T02:44:03,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/47644e66f834413298459a9124c65ea2 2024-11-14T02:44:03,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/c8cc9afb2d2e41a88330a7d7ab9c1325 is 50, key is test_row_0/C:col10/1731552241036/Put/seqid=0 2024-11-14T02:44:03,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742036_1212 (size=12301) 2024-11-14T02:44:03,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-14T02:44:03,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:03,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:44:03,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:03,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:03,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:03,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:03,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T02:44:03,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552303194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:03,195 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_2' in 30ff5d65ef696bee83471a2834e374a0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:44:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:03,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45930 deadline: 1731552303253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:03,254 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_0' in 30ff5d65ef696bee83471a2834e374a0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:44:03,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:03,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:03,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:03,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:03,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46018 deadline: 1731552303263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:03,264 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_0' in 30ff5d65ef696bee83471a2834e374a0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:44:03,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:03,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:03,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:03,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552303313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:03,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552303313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:03,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-14T02:44:03,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:03,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:44:03,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:03,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:03,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:03,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/c8cc9afb2d2e41a88330a7d7ab9c1325 2024-11-14T02:44:03,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/a6a3da9ed11f465382ea365b9c941b5f as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/a6a3da9ed11f465382ea365b9c941b5f 2024-11-14T02:44:03,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/a6a3da9ed11f465382ea365b9c941b5f, entries=200, sequenceid=323, filesize=39.0 K 2024-11-14T02:44:03,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/47644e66f834413298459a9124c65ea2 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/47644e66f834413298459a9124c65ea2 2024-11-14T02:44:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/47644e66f834413298459a9124c65ea2, entries=150, sequenceid=323, filesize=12.0 K 2024-11-14T02:44:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/c8cc9afb2d2e41a88330a7d7ab9c1325 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/c8cc9afb2d2e41a88330a7d7ab9c1325 2024-11-14T02:44:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-14T02:44:03,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:03,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:44:03,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:03,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:03,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/c8cc9afb2d2e41a88330a7d7ab9c1325, entries=150, sequenceid=323, filesize=12.0 K 2024-11-14T02:44:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~201.27 KB/206100, heapSize ~528.05 KB/540720, currentSize=0 B/0 for 30ff5d65ef696bee83471a2834e374a0 in 1306ms, sequenceid=323, compaction requested=true 2024-11-14T02:44:03,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:44:03,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30ff5d65ef696bee83471a2834e374a0:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:44:03,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:03,481 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:03,481 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:03,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30ff5d65ef696bee83471a2834e374a0:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:44:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:03,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30ff5d65ef696bee83471a2834e374a0:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:44:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,483 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32631 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:03,483 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 30ff5d65ef696bee83471a2834e374a0/B is initiating minor compaction (all files) 2024-11-14T02:44:03,483 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30ff5d65ef696bee83471a2834e374a0/B in TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:03,483 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/5afa504038a4445c8afacffe193ecf91, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/57eb4eb016fe42dba767370b2c29e6a1, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/47644e66f834413298459a9124c65ea2] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp, totalSize=31.9 K 2024-11-14T02:44:03,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,483 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94377 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:03,483 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 30ff5d65ef696bee83471a2834e374a0/A is initiating minor compaction (all files) 2024-11-14T02:44:03,483 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30ff5d65ef696bee83471a2834e374a0/A in TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:03,483 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/99f166cdcf28459c8401eb55da46949a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/1b68e6d8366147be9fb06dcd74ef037a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/a6a3da9ed11f465382ea365b9c941b5f] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp, totalSize=92.2 K 2024-11-14T02:44:03,483 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:03,484 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. files: [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/99f166cdcf28459c8401eb55da46949a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/1b68e6d8366147be9fb06dcd74ef037a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/a6a3da9ed11f465382ea365b9c941b5f] 2024-11-14T02:44:03,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,484 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 5afa504038a4445c8afacffe193ecf91, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731552239070 2024-11-14T02:44:03,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,484 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99f166cdcf28459c8401eb55da46949a, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731552239070 2024-11-14T02:44:03,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,485 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 57eb4eb016fe42dba767370b2c29e6a1, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731552240856 2024-11-14T02:44:03,485 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b68e6d8366147be9fb06dcd74ef037a, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731552240853 2024-11-14T02:44:03,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,485 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6a3da9ed11f465382ea365b9c941b5f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731552241036 2024-11-14T02:44:03,485 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 47644e66f834413298459a9124c65ea2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731552241036 2024-11-14T02:44:03,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,492 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:44:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,494 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411140817739dc1654df9a81991c282538260_30ff5d65ef696bee83471a2834e374a0 store=[table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:44:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,495 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30ff5d65ef696bee83471a2834e374a0#B#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,495 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/b5264ffe4f2d461398e58260178dc0e5 is 50, key is test_row_0/B:col10/1731552241036/Put/seqid=0 2024-11-14T02:44:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,496 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411140817739dc1654df9a81991c282538260_30ff5d65ef696bee83471a2834e374a0, store=[table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:44:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,496 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140817739dc1654df9a81991c282538260_30ff5d65ef696bee83471a2834e374a0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:44:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742038_1214 (size=13017) 2024-11-14T02:44:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,531 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/b5264ffe4f2d461398e58260178dc0e5 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/b5264ffe4f2d461398e58260178dc0e5 2024-11-14T02:44:03,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742037_1213 (size=4469) 2024-11-14T02:44:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,536 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30ff5d65ef696bee83471a2834e374a0#A#compaction#176 average throughput is 0.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,536 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/8ec1766cc21b4a3fa70e775367078e5e is 175, key is test_row_0/A:col10/1731552241036/Put/seqid=0 2024-11-14T02:44:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,541 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30ff5d65ef696bee83471a2834e374a0/B of 30ff5d65ef696bee83471a2834e374a0 into b5264ffe4f2d461398e58260178dc0e5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:03,541 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:44:03,541 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., storeName=30ff5d65ef696bee83471a2834e374a0/B, priority=13, startTime=1731552243481; duration=0sec 2024-11-14T02:44:03,541 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:03,541 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30ff5d65ef696bee83471a2834e374a0:B 2024-11-14T02:44:03,541 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:03,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,543 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32631 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:03,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,543 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 30ff5d65ef696bee83471a2834e374a0/C is initiating minor compaction (all files) 2024-11-14T02:44:03,543 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30ff5d65ef696bee83471a2834e374a0/C in TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:03,543 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/3f457c989fc24539b398857872f61557, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/eca274203799419299e14cd435c8434d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/c8cc9afb2d2e41a88330a7d7ab9c1325] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp, totalSize=31.9 K 2024-11-14T02:44:03,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,543 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f457c989fc24539b398857872f61557, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731552239070 2024-11-14T02:44:03,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,544 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting eca274203799419299e14cd435c8434d, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731552240856 2024-11-14T02:44:03,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,544 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting c8cc9afb2d2e41a88330a7d7ab9c1325, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731552241036 2024-11-14T02:44:03,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,561 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30ff5d65ef696bee83471a2834e374a0#C#compaction#178 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:03,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,562 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/5017740f42f043fd9af5e2e2a149724c is 50, key is test_row_0/C:col10/1731552241036/Put/seqid=0 2024-11-14T02:44:03,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742039_1215 (size=31971) 2024-11-14T02:44:03,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742040_1216 (size=13017) 2024-11-14T02:44:03,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,580 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/5017740f42f043fd9af5e2e2a149724c as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/5017740f42f043fd9af5e2e2a149724c 2024-11-14T02:44:03,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,588 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30ff5d65ef696bee83471a2834e374a0/C of 30ff5d65ef696bee83471a2834e374a0 into 5017740f42f043fd9af5e2e2a149724c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,588 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:44:03,588 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., storeName=30ff5d65ef696bee83471a2834e374a0/C, priority=13, startTime=1731552243482; duration=0sec 2024-11-14T02:44:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,588 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:03,588 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30ff5d65ef696bee83471a2834e374a0:C 2024-11-14T02:44:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-14T02:44:03,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:03,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:44:03,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:03,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-14T02:44:03,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-14T02:44:03,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,634 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-14T02:44:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,634 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0750 sec 2024-11-14T02:44:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.0800 sec 2024-11-14T02:44:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T02:44:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,693 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,694 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-14T02:44:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,696 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T02:44:03,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,697 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:03,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,697 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:03,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T02:44:03,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T02:44:03,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:03,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:44:03,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:03,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-14T02:44:03,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-14T02:44:03,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,853 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-14T02:44:03,853 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-14T02:44:03,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,856 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 160 msec 2024-11-14T02:44:03,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,978 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/8ec1766cc21b4a3fa70e775367078e5e as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/8ec1766cc21b4a3fa70e775367078e5e 2024-11-14T02:44:03,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,984 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30ff5d65ef696bee83471a2834e374a0/A of 30ff5d65ef696bee83471a2834e374a0 into 8ec1766cc21b4a3fa70e775367078e5e(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:03,984 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:44:03,984 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., storeName=30ff5d65ef696bee83471a2834e374a0/A, priority=13, startTime=1731552243481; duration=0sec 2024-11-14T02:44:03,984 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:03,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,984 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30ff5d65ef696bee83471a2834e374a0:A 2024-11-14T02:44:03,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:03,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T02:44:04,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,014 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:04,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,014 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:04,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-14T02:44:04,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T02:44:04,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,017 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:04,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,018 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:04,018 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:04,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T02:44:04,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-14T02:44:04,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:04,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:44:04,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:04,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-14T02:44:04,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-14T02:44:04,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-14T02:44:04,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-14T02:44:04,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 161 msec 2024-11-14T02:44:04,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T02:44:04,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,334 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:04,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,334 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:04,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-14T02:44:04,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,337 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:04,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T02:44:04,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,337 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:04,338 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:04,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:04,366 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:44:04,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30ff5d65ef696bee83471a2834e374a0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T02:44:04,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114726f1c6d84eb4458904e6e415ef3a8af_30ff5d65ef696bee83471a2834e374a0 is 50, key is test_row_0/A:col10/1731552244352/Put/seqid=0 2024-11-14T02:44:04,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742042_1218 (size=25158) 2024-11-14T02:44:04,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T02:44:04,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:04,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552304444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:04,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:04,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:04,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:04,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:04,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552304448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:04,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:04,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:04,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:04,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T02:44:04,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:04,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:44:04,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:04,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:04,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:04,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:04,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:04,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552304553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:04,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:04,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:04,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:04,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:04,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552304563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:04,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:04,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:04,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:04,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T02:44:04,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:04,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:44:04,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:04,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:04,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:04,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:04,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T02:44:04,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:04,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552304763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:04,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:04,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:04,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:04,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:04,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552304773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:04,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:04,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:04,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:04,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T02:44:04,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:04,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:44:04,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:04,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:04,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:04,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:04,802 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,806 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114726f1c6d84eb4458904e6e415ef3a8af_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114726f1c6d84eb4458904e6e415ef3a8af_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:04,807 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/0c9a8059ac2441ec87f11884e601a560, store: [table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:44:04,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/0c9a8059ac2441ec87f11884e601a560 is 175, key is test_row_0/A:col10/1731552244352/Put/seqid=0 2024-11-14T02:44:04,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742041_1217 (size=74795) 2024-11-14T02:44:04,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T02:44:04,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:04,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:44:04,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:04,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:04,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:04,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:04,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T02:44:05,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:05,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552305073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:05,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:05,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:05,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:05,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552305083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:05,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:05,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:05,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:05,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T02:44:05,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:05,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:44:05,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:05,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:05,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:05,211 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=340, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/0c9a8059ac2441ec87f11884e601a560 2024-11-14T02:44:05,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:05,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46004 deadline: 1731552305213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:05,214 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_2' in 30ff5d65ef696bee83471a2834e374a0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:44:05,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:05,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:05,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:05,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/2de22d354e094aa69b0b2077b554f195 is 50, key is test_row_0/B:col10/1731552244352/Put/seqid=0 2024-11-14T02:44:05,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742043_1219 (size=12301) 2024-11-14T02:44:05,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T02:44:05,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:05,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:44:05,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:05,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:05,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:05,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T02:44:05,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:05,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:44:05,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:05,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:05,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:05,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:05,462 INFO [Thread-655 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-655. 2024-11-14T02:44:05,462 DEBUG [Thread-655 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:05,462 DEBUG [Thread-655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:05,462 DEBUG [Thread-655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:05,462 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:05,463 INFO [Thread-657 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-657. 2024-11-14T02:44:05,463 DEBUG [Thread-657 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:05,463 DEBUG [Thread-657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:05,463 DEBUG [Thread-657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:05,463 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:05,464 INFO [Thread-653 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-653. 2024-11-14T02:44:05,464 DEBUG [Thread-653 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:05,464 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:05,464 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:05,464 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:05,466 INFO [Thread-659 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-659. 2024-11-14T02:44:05,466 DEBUG [Thread-659 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:05,466 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:05,467 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:05,467 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:05,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T02:44:05,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T02:44:05,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:05,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:44:05,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:05,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:05,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:05,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45948 deadline: 1731552305584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:05,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:05,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45900 deadline: 1731552305593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:05,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:05,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:05,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:05,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/2de22d354e094aa69b0b2077b554f195 2024-11-14T02:44:05,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/aa835009e1b14683ba67cc4343cf93ba is 50, key is test_row_0/C:col10/1731552244352/Put/seqid=0 2024-11-14T02:44:05,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742044_1220 (size=12301) 2024-11-14T02:44:05,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T02:44:05,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:05,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:44:05,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:05,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:05,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:05,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T02:44:05,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:05,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:44:05,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:05,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:05,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:06,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T02:44:06,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:06,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:44:06,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:06,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:06,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:06,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/aa835009e1b14683ba67cc4343cf93ba 2024-11-14T02:44:06,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/0c9a8059ac2441ec87f11884e601a560 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/0c9a8059ac2441ec87f11884e601a560 2024-11-14T02:44:06,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/0c9a8059ac2441ec87f11884e601a560, entries=400, sequenceid=340, filesize=73.0 K 2024-11-14T02:44:06,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/2de22d354e094aa69b0b2077b554f195 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/2de22d354e094aa69b0b2077b554f195 2024-11-14T02:44:06,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/2de22d354e094aa69b0b2077b554f195, entries=150, sequenceid=340, filesize=12.0 K 2024-11-14T02:44:06,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/aa835009e1b14683ba67cc4343cf93ba as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/aa835009e1b14683ba67cc4343cf93ba 2024-11-14T02:44:06,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/aa835009e1b14683ba67cc4343cf93ba, entries=150, sequenceid=340, filesize=12.0 K 2024-11-14T02:44:06,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 30ff5d65ef696bee83471a2834e374a0 in 1710ms, sequenceid=340, compaction requested=false 2024-11-14T02:44:06,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:44:06,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T02:44:06,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:06,180 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 30ff5d65ef696bee83471a2834e374a0 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T02:44:06,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114065b9090d5a54d489b1dc1ebd3a3cd74_30ff5d65ef696bee83471a2834e374a0 is 50, key is test_row_0/A:col10/1731552244430/Put/seqid=0 2024-11-14T02:44:06,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742045_1221 (size=12454) 2024-11-14T02:44:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T02:44:06,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:06,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:06,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. as already flushing 2024-11-14T02:44:06,597 INFO [Thread-642 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-642. 2024-11-14T02:44:06,597 DEBUG [Thread-642 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:06,597 DEBUG [Thread-642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:06,598 DEBUG [Thread-642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:06,598 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:06,602 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114065b9090d5a54d489b1dc1ebd3a3cd74_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114065b9090d5a54d489b1dc1ebd3a3cd74_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:06,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/edef83b604ff4302ab72cb94eb267029, store: [table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:44:06,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/edef83b604ff4302ab72cb94eb267029 is 175, key is test_row_0/A:col10/1731552244430/Put/seqid=0 2024-11-14T02:44:06,605 INFO [Thread-648 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-648. 2024-11-14T02:44:06,605 DEBUG [Thread-648 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:06,605 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:06,605 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:06,605 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:06,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742046_1222 (size=31255) 2024-11-14T02:44:07,009 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=365, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/edef83b604ff4302ab72cb94eb267029 2024-11-14T02:44:07,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/e813c5e4b4614ef39ae79fae45000458 is 50, key is test_row_0/B:col10/1731552244430/Put/seqid=0 2024-11-14T02:44:07,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742047_1223 (size=12301) 2024-11-14T02:44:07,295 INFO [Thread-650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-650. 2024-11-14T02:44:07,295 INFO [Thread-646 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-646. 2024-11-14T02:44:07,295 DEBUG [Thread-646 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:07,295 DEBUG [Thread-650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:07,295 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:07,295 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:07,296 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:07,296 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:07,296 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:07,296 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:07,423 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/e813c5e4b4614ef39ae79fae45000458 2024-11-14T02:44:07,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/a404d8c27810441dab2e532c5fe8a4e1 is 50, key is test_row_0/C:col10/1731552244430/Put/seqid=0 2024-11-14T02:44:07,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742048_1224 (size=12301) 2024-11-14T02:44:07,836 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/a404d8c27810441dab2e532c5fe8a4e1 2024-11-14T02:44:07,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/edef83b604ff4302ab72cb94eb267029 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/edef83b604ff4302ab72cb94eb267029 2024-11-14T02:44:07,853 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/edef83b604ff4302ab72cb94eb267029, entries=150, sequenceid=365, filesize=30.5 K 2024-11-14T02:44:07,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/e813c5e4b4614ef39ae79fae45000458 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/e813c5e4b4614ef39ae79fae45000458 2024-11-14T02:44:07,859 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/e813c5e4b4614ef39ae79fae45000458, entries=150, sequenceid=365, filesize=12.0 K 2024-11-14T02:44:07,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/a404d8c27810441dab2e532c5fe8a4e1 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/a404d8c27810441dab2e532c5fe8a4e1 2024-11-14T02:44:07,864 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/a404d8c27810441dab2e532c5fe8a4e1, entries=150, sequenceid=365, filesize=12.0 K 2024-11-14T02:44:07,864 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=26.84 KB/27480 for 30ff5d65ef696bee83471a2834e374a0 in 1684ms, sequenceid=365, compaction requested=true 2024-11-14T02:44:07,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 30ff5d65ef696bee83471a2834e374a0: 2024-11-14T02:44:07,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:07,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-14T02:44:07,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-14T02:44:07,867 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-14T02:44:07,867 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5280 sec 2024-11-14T02:44:07,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 3.5340 sec 2024-11-14T02:44:08,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T02:44:08,514 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:09,248 INFO [Thread-644 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-644. 2024-11-14T02:44:09,248 DEBUG [Thread-644 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:09,248 DEBUG [Thread-644 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:09,249 DEBUG [Thread-644 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:09,249 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:09,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T02:44:09,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-14T02:44:09,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-14T02:44:09,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-14T02:44:09,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-14T02:44:09,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-14T02:44:09,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T02:44:09,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4004 2024-11-14T02:44:09,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3898 2024-11-14T02:44:09,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T02:44:09,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1660 2024-11-14T02:44:09,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4977 rows 2024-11-14T02:44:09,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1664 2024-11-14T02:44:09,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4989 rows 2024-11-14T02:44:09,250 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T02:44:09,250 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T02:44:09,251 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:09,251 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:09,251 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:09,253 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T02:44:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T02:44:09,258 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552249257"}]},"ts":"1731552249257"} 2024-11-14T02:44:09,260 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T02:44:09,260 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T02:44:09,261 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T02:44:09,262 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=30ff5d65ef696bee83471a2834e374a0, UNASSIGN}] 2024-11-14T02:44:09,263 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=30ff5d65ef696bee83471a2834e374a0, UNASSIGN 2024-11-14T02:44:09,264 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=30ff5d65ef696bee83471a2834e374a0, regionState=CLOSING, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:09,265 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43239 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=CLOSING, location=83c5a5c119d5,41455,1731552191259, table=TestAcidGuarantees, region=30ff5d65ef696bee83471a2834e374a0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-14T02:44:09,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=30ff5d65ef696bee83471a2834e374a0, UNASSIGN because future has completed 2024-11-14T02:44:09,266 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T02:44:09,266 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259}] 2024-11-14T02:44:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T02:44:09,420 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(122): Close 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:09,421 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T02:44:09,421 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1722): Closing 30ff5d65ef696bee83471a2834e374a0, disabling compactions & flushes 2024-11-14T02:44:09,421 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:09,421 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:09,421 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. after waiting 0 ms 2024-11-14T02:44:09,421 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:09,422 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(2902): Flushing 30ff5d65ef696bee83471a2834e374a0 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-14T02:44:09,432 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149ce8c90c694e4f1b9352c9e63cce8b50_30ff5d65ef696bee83471a2834e374a0 is 50, key is test_row_0/A:col10/1731552247294/Put/seqid=0 2024-11-14T02:44:09,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742049_1225 (size=12454) 2024-11-14T02:44:09,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T02:44:09,749 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T02:44:09,840 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:09,851 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149ce8c90c694e4f1b9352c9e63cce8b50_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149ce8c90c694e4f1b9352c9e63cce8b50_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:09,851 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/3345bc5558d84522b392b799b8f1c618, store: [table=TestAcidGuarantees family=A region=30ff5d65ef696bee83471a2834e374a0] 2024-11-14T02:44:09,852 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/3345bc5558d84522b392b799b8f1c618 is 175, key is test_row_0/A:col10/1731552247294/Put/seqid=0 2024-11-14T02:44:09,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742050_1226 (size=31255) 2024-11-14T02:44:09,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T02:44:10,257 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=373, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/3345bc5558d84522b392b799b8f1c618 2024-11-14T02:44:10,272 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/7cce89f1312745ca81efdae355ce7e96 is 50, key is test_row_0/B:col10/1731552247294/Put/seqid=0 2024-11-14T02:44:10,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742051_1227 (size=12301) 2024-11-14T02:44:10,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T02:44:10,677 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/7cce89f1312745ca81efdae355ce7e96 2024-11-14T02:44:10,690 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/9f5e3cc8543c489589a6041baeb0bd03 is 50, key is test_row_0/C:col10/1731552247294/Put/seqid=0 2024-11-14T02:44:10,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742052_1228 (size=12301) 2024-11-14T02:44:11,096 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/9f5e3cc8543c489589a6041baeb0bd03 2024-11-14T02:44:11,108 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/A/3345bc5558d84522b392b799b8f1c618 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/3345bc5558d84522b392b799b8f1c618 2024-11-14T02:44:11,116 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/3345bc5558d84522b392b799b8f1c618, entries=150, sequenceid=373, filesize=30.5 K 2024-11-14T02:44:11,117 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/B/7cce89f1312745ca81efdae355ce7e96 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/7cce89f1312745ca81efdae355ce7e96 2024-11-14T02:44:11,121 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/7cce89f1312745ca81efdae355ce7e96, entries=150, sequenceid=373, filesize=12.0 K 2024-11-14T02:44:11,122 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/.tmp/C/9f5e3cc8543c489589a6041baeb0bd03 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/9f5e3cc8543c489589a6041baeb0bd03 2024-11-14T02:44:11,125 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/9f5e3cc8543c489589a6041baeb0bd03, entries=150, sequenceid=373, filesize=12.0 K 2024-11-14T02:44:11,126 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 30ff5d65ef696bee83471a2834e374a0 in 1705ms, sequenceid=373, compaction requested=true 2024-11-14T02:44:11,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/2683c858f1d04f06ab5e954875466f14, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/088e7c2877494f00bd2227feb036e915, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/db500ab5e1df494d86929c4ab87e28c1, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/1fa24a5d86504c85b8feb961ace0ee4d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/0a11afe80ff64a3086639ac7b3d07101, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/f2462c25f9314afb8b8151fad5265132, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/dca56e5935e94b749d15f528727fd627, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/52c1bfba06c547398b0d663070eeb0ca, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/551a9a4ae4ec4dfc82539592d62f0532, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/cba105f65c924f29a283b0a20f35caa0, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/3479877f754348c28f988dbc241b7d62, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/269e001169a4419989145f5a9320e40f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/9eeca47c76454916af4168e1ce28c9e1, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/bf69996fef5244e4994406f4b6f36433, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/4edd48534875478c8a4c126a42dd13ec, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/4abf3314e1584f7aaff26f99d107d763, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/8c93ab4ae1ba4a9b8e986d0a6e395d17, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/fbd5da81db5946e9a8b4aa4674e215e0, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/99f166cdcf28459c8401eb55da46949a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/1b68e6d8366147be9fb06dcd74ef037a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/a6a3da9ed11f465382ea365b9c941b5f] to archive 2024-11-14T02:44:11,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:44:11,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/2683c858f1d04f06ab5e954875466f14 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/2683c858f1d04f06ab5e954875466f14 2024-11-14T02:44:11,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/088e7c2877494f00bd2227feb036e915 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/088e7c2877494f00bd2227feb036e915 2024-11-14T02:44:11,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/db500ab5e1df494d86929c4ab87e28c1 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/db500ab5e1df494d86929c4ab87e28c1 2024-11-14T02:44:11,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/1fa24a5d86504c85b8feb961ace0ee4d to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/1fa24a5d86504c85b8feb961ace0ee4d 2024-11-14T02:44:11,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/0a11afe80ff64a3086639ac7b3d07101 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/0a11afe80ff64a3086639ac7b3d07101 2024-11-14T02:44:11,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/f2462c25f9314afb8b8151fad5265132 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/f2462c25f9314afb8b8151fad5265132 2024-11-14T02:44:11,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/dca56e5935e94b749d15f528727fd627 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/dca56e5935e94b749d15f528727fd627 2024-11-14T02:44:11,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/52c1bfba06c547398b0d663070eeb0ca to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/52c1bfba06c547398b0d663070eeb0ca 2024-11-14T02:44:11,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/551a9a4ae4ec4dfc82539592d62f0532 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/551a9a4ae4ec4dfc82539592d62f0532 2024-11-14T02:44:11,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/cba105f65c924f29a283b0a20f35caa0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/cba105f65c924f29a283b0a20f35caa0 2024-11-14T02:44:11,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/3479877f754348c28f988dbc241b7d62 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/3479877f754348c28f988dbc241b7d62 2024-11-14T02:44:11,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/269e001169a4419989145f5a9320e40f to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/269e001169a4419989145f5a9320e40f 2024-11-14T02:44:11,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/9eeca47c76454916af4168e1ce28c9e1 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/9eeca47c76454916af4168e1ce28c9e1 2024-11-14T02:44:11,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/bf69996fef5244e4994406f4b6f36433 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/bf69996fef5244e4994406f4b6f36433 2024-11-14T02:44:11,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/4edd48534875478c8a4c126a42dd13ec to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/4edd48534875478c8a4c126a42dd13ec 2024-11-14T02:44:11,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/4abf3314e1584f7aaff26f99d107d763 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/4abf3314e1584f7aaff26f99d107d763 2024-11-14T02:44:11,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/8c93ab4ae1ba4a9b8e986d0a6e395d17 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/8c93ab4ae1ba4a9b8e986d0a6e395d17 2024-11-14T02:44:11,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/fbd5da81db5946e9a8b4aa4674e215e0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/fbd5da81db5946e9a8b4aa4674e215e0 2024-11-14T02:44:11,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/99f166cdcf28459c8401eb55da46949a to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/99f166cdcf28459c8401eb55da46949a 2024-11-14T02:44:11,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/1b68e6d8366147be9fb06dcd74ef037a to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/1b68e6d8366147be9fb06dcd74ef037a 2024-11-14T02:44:11,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/a6a3da9ed11f465382ea365b9c941b5f to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/a6a3da9ed11f465382ea365b9c941b5f 2024-11-14T02:44:11,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/fafbd5da692241dca9b56739ffbca99f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/3efdfd8e34b44e53a6c7fb40491ded01, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/0d294997d59e4f1b85abb2304e47c74e, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/c33442bae8354bfcb82cec73c0fc4762, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/99b57bf902834728b3a0390c1276b68d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/ab1e6843dca245908288860aa03ed29e, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/80628c31304d46baa19157487156ab75, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/b0988fe5cc704184b40d2084937e8e28, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/926444e146b6498daa03e8ba39e5d17b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/35e8c183292741ffb5411427d08a3ea1, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/e240367aec0047989d7b0cdd9cd85d78, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/c893084ce02f466f85f6c688c3868e29, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/7b31254c0ade461f85ef4f79113d3235, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/b5335804eee041489968c7fe284b66cf, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/3e4933ab52624073a7b87a955ceff00f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/ba4106b2dbf243f891e9ee6a26b3f3c4, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/a20ebd32a3ea43aa9848cbc3593bc662, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/5afa504038a4445c8afacffe193ecf91, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/2929cfb0d0df4c48adec27478eb9661a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/57eb4eb016fe42dba767370b2c29e6a1, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/47644e66f834413298459a9124c65ea2] to archive 2024-11-14T02:44:11,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:44:11,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/fafbd5da692241dca9b56739ffbca99f to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/fafbd5da692241dca9b56739ffbca99f 2024-11-14T02:44:11,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/3efdfd8e34b44e53a6c7fb40491ded01 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/3efdfd8e34b44e53a6c7fb40491ded01 2024-11-14T02:44:11,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/0d294997d59e4f1b85abb2304e47c74e to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/0d294997d59e4f1b85abb2304e47c74e 2024-11-14T02:44:11,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/c33442bae8354bfcb82cec73c0fc4762 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/c33442bae8354bfcb82cec73c0fc4762 2024-11-14T02:44:11,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/99b57bf902834728b3a0390c1276b68d to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/99b57bf902834728b3a0390c1276b68d 2024-11-14T02:44:11,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/ab1e6843dca245908288860aa03ed29e to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/ab1e6843dca245908288860aa03ed29e 2024-11-14T02:44:11,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/80628c31304d46baa19157487156ab75 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/80628c31304d46baa19157487156ab75 2024-11-14T02:44:11,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/b0988fe5cc704184b40d2084937e8e28 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/b0988fe5cc704184b40d2084937e8e28 2024-11-14T02:44:11,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/926444e146b6498daa03e8ba39e5d17b to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/926444e146b6498daa03e8ba39e5d17b 2024-11-14T02:44:11,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/35e8c183292741ffb5411427d08a3ea1 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/35e8c183292741ffb5411427d08a3ea1 2024-11-14T02:44:11,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/e240367aec0047989d7b0cdd9cd85d78 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/e240367aec0047989d7b0cdd9cd85d78 2024-11-14T02:44:11,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/c893084ce02f466f85f6c688c3868e29 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/c893084ce02f466f85f6c688c3868e29 2024-11-14T02:44:11,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/7b31254c0ade461f85ef4f79113d3235 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/7b31254c0ade461f85ef4f79113d3235 2024-11-14T02:44:11,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/b5335804eee041489968c7fe284b66cf to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/b5335804eee041489968c7fe284b66cf 2024-11-14T02:44:11,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/3e4933ab52624073a7b87a955ceff00f to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/3e4933ab52624073a7b87a955ceff00f 2024-11-14T02:44:11,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/ba4106b2dbf243f891e9ee6a26b3f3c4 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/ba4106b2dbf243f891e9ee6a26b3f3c4 2024-11-14T02:44:11,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/a20ebd32a3ea43aa9848cbc3593bc662 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/a20ebd32a3ea43aa9848cbc3593bc662 2024-11-14T02:44:11,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/5afa504038a4445c8afacffe193ecf91 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/5afa504038a4445c8afacffe193ecf91 2024-11-14T02:44:11,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/2929cfb0d0df4c48adec27478eb9661a to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/2929cfb0d0df4c48adec27478eb9661a 2024-11-14T02:44:11,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/57eb4eb016fe42dba767370b2c29e6a1 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/57eb4eb016fe42dba767370b2c29e6a1 2024-11-14T02:44:11,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/47644e66f834413298459a9124c65ea2 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/47644e66f834413298459a9124c65ea2 2024-11-14T02:44:11,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/048f67ce9a8b46d697bd0d90fb1efb04, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/20c21f212c6a47348cced829fce0e398, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/1bfc5541fe794f13b4fcec03eee8d40a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/8310e61eb9a94e8a87f09ad8f26a18c9, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/eb84132149704060ad64d038d599fec1, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/1fde2989bfb443d09fced3c3552abcc6, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/e95d4817c038457dadf2a7620911821c, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/f952d17a299c4c78b13a08c7e437127c, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/a9183dca9c144216973bb7e0354c0157, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/4fcc4358ef834ff2935e99ac5be9b43c, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/e7e1dd5f04b24f67bfb159108929ae38, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/0a7bfc92b41c43549fb39ef3edc0033e, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/7b7e6a25b3fd46fbbee3db5451710351, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/cc978d3be2e345cd9a09f4c40a54b145, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/9094835098ac4df5a093bb3aa35cc548, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/e770049053e04a41bec5b1145f3a6af6, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/495cc90bb3fe4eac8bceb675dc022067, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/3f457c989fc24539b398857872f61557, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/21bb355770664c91a4e8c34bf054a961, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/eca274203799419299e14cd435c8434d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/c8cc9afb2d2e41a88330a7d7ab9c1325] to archive 2024-11-14T02:44:11,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:44:11,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/048f67ce9a8b46d697bd0d90fb1efb04 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/048f67ce9a8b46d697bd0d90fb1efb04 2024-11-14T02:44:11,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/20c21f212c6a47348cced829fce0e398 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/20c21f212c6a47348cced829fce0e398 2024-11-14T02:44:11,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/1bfc5541fe794f13b4fcec03eee8d40a to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/1bfc5541fe794f13b4fcec03eee8d40a 2024-11-14T02:44:11,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/8310e61eb9a94e8a87f09ad8f26a18c9 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/8310e61eb9a94e8a87f09ad8f26a18c9 2024-11-14T02:44:11,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/eb84132149704060ad64d038d599fec1 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/eb84132149704060ad64d038d599fec1 2024-11-14T02:44:11,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/1fde2989bfb443d09fced3c3552abcc6 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/1fde2989bfb443d09fced3c3552abcc6 2024-11-14T02:44:11,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/e95d4817c038457dadf2a7620911821c to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/e95d4817c038457dadf2a7620911821c 2024-11-14T02:44:11,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/f952d17a299c4c78b13a08c7e437127c to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/f952d17a299c4c78b13a08c7e437127c 2024-11-14T02:44:11,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/a9183dca9c144216973bb7e0354c0157 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/a9183dca9c144216973bb7e0354c0157 2024-11-14T02:44:11,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/4fcc4358ef834ff2935e99ac5be9b43c to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/4fcc4358ef834ff2935e99ac5be9b43c 2024-11-14T02:44:11,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/e7e1dd5f04b24f67bfb159108929ae38 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/e7e1dd5f04b24f67bfb159108929ae38 2024-11-14T02:44:11,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/0a7bfc92b41c43549fb39ef3edc0033e to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/0a7bfc92b41c43549fb39ef3edc0033e 2024-11-14T02:44:11,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/7b7e6a25b3fd46fbbee3db5451710351 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/7b7e6a25b3fd46fbbee3db5451710351 2024-11-14T02:44:11,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/cc978d3be2e345cd9a09f4c40a54b145 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/cc978d3be2e345cd9a09f4c40a54b145 2024-11-14T02:44:11,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/9094835098ac4df5a093bb3aa35cc548 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/9094835098ac4df5a093bb3aa35cc548 2024-11-14T02:44:11,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/e770049053e04a41bec5b1145f3a6af6 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/e770049053e04a41bec5b1145f3a6af6 2024-11-14T02:44:11,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/495cc90bb3fe4eac8bceb675dc022067 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/495cc90bb3fe4eac8bceb675dc022067 2024-11-14T02:44:11,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/3f457c989fc24539b398857872f61557 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/3f457c989fc24539b398857872f61557 2024-11-14T02:44:11,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/21bb355770664c91a4e8c34bf054a961 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/21bb355770664c91a4e8c34bf054a961 2024-11-14T02:44:11,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/eca274203799419299e14cd435c8434d to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/eca274203799419299e14cd435c8434d 2024-11-14T02:44:11,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/c8cc9afb2d2e41a88330a7d7ab9c1325 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/c8cc9afb2d2e41a88330a7d7ab9c1325 2024-11-14T02:44:11,195 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/recovered.edits/376.seqid, newMaxSeqId=376, maxSeqId=4 2024-11-14T02:44:11,196 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. 2024-11-14T02:44:11,196 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1676): Region close journal for 30ff5d65ef696bee83471a2834e374a0: Waiting for close lock at 1731552249421Running coprocessor pre-close hooks at 1731552249421Disabling compacts and flushes for region at 1731552249421Disabling writes for close at 1731552249421Obtaining lock to block concurrent updates at 1731552249422 (+1 ms)Preparing flush snapshotting stores in 30ff5d65ef696bee83471a2834e374a0 at 1731552249422Finished memstore snapshotting TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731552249422Flushing stores of TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0. at 1731552249424 (+2 ms)Flushing 30ff5d65ef696bee83471a2834e374a0/A: creating writer at 1731552249424Flushing mob file 30ff5d65ef696bee83471a2834e374a0/A: appending metadata at 1731552249432 (+8 ms)Flushing mob file 30ff5d65ef696bee83471a2834e374a0/A: closing flushed file at 1731552249432Flushing 30ff5d65ef696bee83471a2834e374a0/A: appending metadata at 1731552249852 (+420 ms)Flushing 30ff5d65ef696bee83471a2834e374a0/A: closing flushed file at 1731552249852Flushing 30ff5d65ef696bee83471a2834e374a0/B: creating writer at 1731552250266 (+414 ms)Flushing 30ff5d65ef696bee83471a2834e374a0/B: appending metadata at 1731552250271 (+5 ms)Flushing 30ff5d65ef696bee83471a2834e374a0/B: closing flushed file at 1731552250271Flushing 30ff5d65ef696bee83471a2834e374a0/C: creating writer at 1731552250685 (+414 ms)Flushing 30ff5d65ef696bee83471a2834e374a0/C: appending metadata at 1731552250690 (+5 ms)Flushing 30ff5d65ef696bee83471a2834e374a0/C: closing flushed file at 1731552250690Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8afc12c: reopening flushed file at 1731552251107 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c4c9d22: reopening flushed file at 1731552251116 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58289771: reopening flushed file at 1731552251121 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 30ff5d65ef696bee83471a2834e374a0 in 1705ms, sequenceid=373, compaction requested=true at 1731552251126 (+5 ms)Writing region close event to WAL at 1731552251193 (+67 ms)Running coprocessor post-close hooks at 1731552251196 (+3 ms)Closed at 1731552251196 2024-11-14T02:44:11,198 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(157): Closed 30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:11,198 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=30ff5d65ef696bee83471a2834e374a0, regionState=CLOSED 2024-11-14T02:44:11,199 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 because future has completed 2024-11-14T02:44:11,201 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-14T02:44:11,201 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; CloseRegionProcedure 30ff5d65ef696bee83471a2834e374a0, server=83c5a5c119d5,41455,1731552191259 in 1.9340 sec 2024-11-14T02:44:11,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-14T02:44:11,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=30ff5d65ef696bee83471a2834e374a0, UNASSIGN in 1.9390 sec 2024-11-14T02:44:11,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-14T02:44:11,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9430 sec 2024-11-14T02:44:11,205 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552251205"}]},"ts":"1731552251205"} 2024-11-14T02:44:11,206 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T02:44:11,206 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T02:44:11,208 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9540 sec 2024-11-14T02:44:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T02:44:11,405 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:11,406 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T02:44:11,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:11,410 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:11,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T02:44:11,412 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=66, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:11,417 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:11,423 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A, FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B, FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C, FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/recovered.edits] 2024-11-14T02:44:11,427 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/0c9a8059ac2441ec87f11884e601a560 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/0c9a8059ac2441ec87f11884e601a560 2024-11-14T02:44:11,429 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/3345bc5558d84522b392b799b8f1c618 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/3345bc5558d84522b392b799b8f1c618 2024-11-14T02:44:11,431 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/8ec1766cc21b4a3fa70e775367078e5e to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/8ec1766cc21b4a3fa70e775367078e5e 2024-11-14T02:44:11,432 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/edef83b604ff4302ab72cb94eb267029 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/A/edef83b604ff4302ab72cb94eb267029 2024-11-14T02:44:11,434 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/2de22d354e094aa69b0b2077b554f195 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/2de22d354e094aa69b0b2077b554f195 2024-11-14T02:44:11,435 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/7cce89f1312745ca81efdae355ce7e96 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/7cce89f1312745ca81efdae355ce7e96 2024-11-14T02:44:11,437 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/b5264ffe4f2d461398e58260178dc0e5 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/b5264ffe4f2d461398e58260178dc0e5 2024-11-14T02:44:11,438 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/e813c5e4b4614ef39ae79fae45000458 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/B/e813c5e4b4614ef39ae79fae45000458 2024-11-14T02:44:11,440 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/5017740f42f043fd9af5e2e2a149724c to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/5017740f42f043fd9af5e2e2a149724c 2024-11-14T02:44:11,441 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/9f5e3cc8543c489589a6041baeb0bd03 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/9f5e3cc8543c489589a6041baeb0bd03 2024-11-14T02:44:11,442 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/a404d8c27810441dab2e532c5fe8a4e1 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/a404d8c27810441dab2e532c5fe8a4e1 2024-11-14T02:44:11,443 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/aa835009e1b14683ba67cc4343cf93ba to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/C/aa835009e1b14683ba67cc4343cf93ba 2024-11-14T02:44:11,445 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/recovered.edits/376.seqid to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0/recovered.edits/376.seqid 2024-11-14T02:44:11,445 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:11,446 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T02:44:11,446 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T02:44:11,447 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T02:44:11,450 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111404aea900beef4fc7994f1cb39572b539_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111404aea900beef4fc7994f1cb39572b539_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:11,451 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114065b9090d5a54d489b1dc1ebd3a3cd74_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114065b9090d5a54d489b1dc1ebd3a3cd74_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:11,452 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143a27086761e7404396327c62c3aaabcb_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143a27086761e7404396327c62c3aaabcb_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:11,453 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144a26c4a3389d487ca389b4331e35f75b_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144a26c4a3389d487ca389b4331e35f75b_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:11,454 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145b98e00d45034f3e8638238d4ef5f6fd_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145b98e00d45034f3e8638238d4ef5f6fd_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:11,455 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146bb8f8935e834726b0603837647386cc_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146bb8f8935e834726b0603837647386cc_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:11,456 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111471a1d8a101ce4a8188b4e32331eb8a2e_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111471a1d8a101ce4a8188b4e32331eb8a2e_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:11,457 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114726f1c6d84eb4458904e6e415ef3a8af_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114726f1c6d84eb4458904e6e415ef3a8af_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:11,458 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148a77d155cb82492b9107f8bfcc831930_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148a77d155cb82492b9107f8bfcc831930_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:11,459 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149ce8c90c694e4f1b9352c9e63cce8b50_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149ce8c90c694e4f1b9352c9e63cce8b50_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:11,459 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a944e65338ed416fb211b80fed9ac149_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a944e65338ed416fb211b80fed9ac149_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:11,460 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114aa1a19e298d144bc85c4e80bb06ce237_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114aa1a19e298d144bc85c4e80bb06ce237_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:11,461 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d2db24f18f2d46d0af8b9946e002989d_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d2db24f18f2d46d0af8b9946e002989d_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:11,462 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d460ae4c3496404a9aedc61d9b9dc661_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d460ae4c3496404a9aedc61d9b9dc661_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:11,462 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d958ee1c9b604d1e89a8fff7b3b46a30_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d958ee1c9b604d1e89a8fff7b3b46a30_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:11,463 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e13607ec63ba4011afad6e9115ea070f_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e13607ec63ba4011afad6e9115ea070f_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:11,464 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e5ab84ff370541558dbc7b263fee89a4_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e5ab84ff370541558dbc7b263fee89a4_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:11,465 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ea47234ca5f3465bb7fd9dacb5ab1c39_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ea47234ca5f3465bb7fd9dacb5ab1c39_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:11,465 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ff7a6cdb041a45f2ac0238d2e6357a0f_30ff5d65ef696bee83471a2834e374a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ff7a6cdb041a45f2ac0238d2e6357a0f_30ff5d65ef696bee83471a2834e374a0 2024-11-14T02:44:11,466 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T02:44:11,468 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=66, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:11,470 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T02:44:11,472 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T02:44:11,473 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=66, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:11,473 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T02:44:11,473 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731552251473"}]},"ts":"9223372036854775807"} 2024-11-14T02:44:11,475 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T02:44:11,475 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 30ff5d65ef696bee83471a2834e374a0, NAME => 'TestAcidGuarantees,,1731552222288.30ff5d65ef696bee83471a2834e374a0.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T02:44:11,475 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T02:44:11,475 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731552251475"}]},"ts":"9223372036854775807"} 2024-11-14T02:44:11,477 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T02:44:11,477 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=66, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:11,478 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 71 msec 2024-11-14T02:44:11,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T02:44:11,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T02:44:11,524 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:11,533 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=242 (was 240) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/cluster_801a70f6-fc7b-44f6-e8da-0c348f54bc49/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/cluster_801a70f6-fc7b-44f6-e8da-0c348f54bc49/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_40334821_22 at /127.0.0.1:35776 [Waiting for operation #1843] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-174932997_22 at /127.0.0.1:52610 [Waiting for operation #1894] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-174932997_22 at /127.0.0.1:40200 [Waiting for operation #1567] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_40334821_22 at /127.0.0.1:52608 [Waiting for operation #1891] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=463 (was 459) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=346 (was 308) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5375 (was 5491) 2024-11-14T02:44:11,543 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=242, OpenFileDescriptor=463, MaxFileDescriptor=1048576, SystemLoadAverage=346, ProcessCount=11, AvailableMemoryMB=5374 2024-11-14T02:44:11,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T02:44:11,544 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T02:44:11,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:11,546 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T02:44:11,547 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:11,547 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 67 2024-11-14T02:44:11,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-14T02:44:11,547 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T02:44:11,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742053_1229 (size=959) 2024-11-14T02:44:11,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-14T02:44:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-14T02:44:11,957 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b 2024-11-14T02:44:11,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742054_1230 (size=53) 2024-11-14T02:44:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-14T02:44:12,368 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:44:12,368 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 35a73edee503a7eb9039857e3cfc97f1, disabling compactions & flushes 2024-11-14T02:44:12,368 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:12,368 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:12,368 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. after waiting 0 ms 2024-11-14T02:44:12,368 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:12,368 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:12,369 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 35a73edee503a7eb9039857e3cfc97f1: Waiting for close lock at 1731552252368Disabling compacts and flushes for region at 1731552252368Disabling writes for close at 1731552252368Writing region close event to WAL at 1731552252368Closed at 1731552252368 2024-11-14T02:44:12,370 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T02:44:12,371 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731552252370"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731552252370"}]},"ts":"1731552252370"} 2024-11-14T02:44:12,372 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T02:44:12,373 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T02:44:12,374 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552252374"}]},"ts":"1731552252374"} 2024-11-14T02:44:12,375 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T02:44:12,376 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=35a73edee503a7eb9039857e3cfc97f1, ASSIGN}] 2024-11-14T02:44:12,377 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=35a73edee503a7eb9039857e3cfc97f1, ASSIGN 2024-11-14T02:44:12,378 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=35a73edee503a7eb9039857e3cfc97f1, ASSIGN; state=OFFLINE, location=83c5a5c119d5,41455,1731552191259; forceNewPlan=false, retain=false 2024-11-14T02:44:12,529 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=35a73edee503a7eb9039857e3cfc97f1, regionState=OPENING, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:12,534 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=35a73edee503a7eb9039857e3cfc97f1, ASSIGN because future has completed 2024-11-14T02:44:12,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259}] 2024-11-14T02:44:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-14T02:44:12,695 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:12,695 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7752): Opening region: {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:44:12,696 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:12,696 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:44:12,696 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7794): checking encryption for 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:12,696 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7797): checking classloading for 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:12,699 INFO [StoreOpener-35a73edee503a7eb9039857e3cfc97f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:12,701 INFO [StoreOpener-35a73edee503a7eb9039857e3cfc97f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 35a73edee503a7eb9039857e3cfc97f1 columnFamilyName A 2024-11-14T02:44:12,701 DEBUG [StoreOpener-35a73edee503a7eb9039857e3cfc97f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:12,702 INFO [StoreOpener-35a73edee503a7eb9039857e3cfc97f1-1 {}] regionserver.HStore(327): Store=35a73edee503a7eb9039857e3cfc97f1/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:44:12,702 INFO [StoreOpener-35a73edee503a7eb9039857e3cfc97f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:12,703 INFO [StoreOpener-35a73edee503a7eb9039857e3cfc97f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 35a73edee503a7eb9039857e3cfc97f1 columnFamilyName B 2024-11-14T02:44:12,703 DEBUG [StoreOpener-35a73edee503a7eb9039857e3cfc97f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:12,704 INFO [StoreOpener-35a73edee503a7eb9039857e3cfc97f1-1 {}] regionserver.HStore(327): Store=35a73edee503a7eb9039857e3cfc97f1/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:44:12,704 INFO [StoreOpener-35a73edee503a7eb9039857e3cfc97f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:12,706 INFO [StoreOpener-35a73edee503a7eb9039857e3cfc97f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 35a73edee503a7eb9039857e3cfc97f1 columnFamilyName C 2024-11-14T02:44:12,706 DEBUG [StoreOpener-35a73edee503a7eb9039857e3cfc97f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:12,706 INFO [StoreOpener-35a73edee503a7eb9039857e3cfc97f1-1 {}] regionserver.HStore(327): Store=35a73edee503a7eb9039857e3cfc97f1/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:44:12,706 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1038): replaying wal for 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:12,707 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:12,708 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:12,709 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1048): stopping wal replay for 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:12,709 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1060): Cleaning up temporary data for 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:12,710 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T02:44:12,712 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1093): writing seq id for 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:12,715 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T02:44:12,715 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1114): Opened 35a73edee503a7eb9039857e3cfc97f1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59726754, jitterRate=-0.11000201106071472}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T02:44:12,715 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:12,716 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1006): Region open journal for 35a73edee503a7eb9039857e3cfc97f1: Running coprocessor pre-open hook at 1731552252696Writing region info on filesystem at 1731552252696Initializing all the Stores at 1731552252699 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552252699Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552252699Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552252699Cleaning up temporary data from old regions at 1731552252709 (+10 ms)Running coprocessor post-open hooks at 1731552252715 (+6 ms)Region opened successfully at 1731552252716 (+1 ms) 2024-11-14T02:44:12,717 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., pid=69, masterSystemTime=1731552252689 2024-11-14T02:44:12,719 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:12,719 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:12,720 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=35a73edee503a7eb9039857e3cfc97f1, regionState=OPEN, openSeqNum=2, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:12,721 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 because future has completed 2024-11-14T02:44:12,723 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-14T02:44:12,723 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 in 188 msec 2024-11-14T02:44:12,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=68, resume processing ppid=67 2024-11-14T02:44:12,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=67, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=35a73edee503a7eb9039857e3cfc97f1, ASSIGN in 347 msec 2024-11-14T02:44:12,725 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T02:44:12,726 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552252725"}]},"ts":"1731552252725"} 2024-11-14T02:44:12,727 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T02:44:12,728 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T02:44:12,729 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1830 sec 2024-11-14T02:44:13,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-14T02:44:13,694 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:13,695 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d8da64d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,695 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:44:13,696 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:13,697 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:44:13,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:13,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:44:13,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a1df49d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:44:13,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:13,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:13,699 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:13,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13e60fc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,701 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:13,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:44:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:13,704 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:13,705 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T02:44:13,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 83c5a5c119d5,43239,1731552190584 2024-11-14T02:44:13,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@448f22f3 2024-11-14T02:44:13,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T02:44:13,706 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T02:44:13,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,707 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:44:13,708 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:13,708 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:44:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:44:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f0c10bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:44:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:13,709 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:13,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32e9b619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,710 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,710 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:44:13,710 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:13,711 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:44:13,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:13,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:44:13,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31aa940, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:44:13,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:13,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:13,712 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:13,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54434fb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,713 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,713 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:44:13,713 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:13,713 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:44:13,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:13,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:44:13,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6c0b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:44:13,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:13,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:13,714 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bf1b307, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,715 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,715 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:44:13,715 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:13,716 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:44:13,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:13,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:44:13,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fdc23e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:44:13,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:13,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:13,717 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:13,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63b66b83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,718 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19903550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,718 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:44:13,718 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:13,719 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:44:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:44:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@593085cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:44:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:13,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:13,720 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:13,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1302306f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,721 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,721 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:44:13,722 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:13,722 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:44:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:13,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:44:13,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67e01e55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:44:13,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:13,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:13,724 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:13,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b670752, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:44:13,725 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:13,726 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:44:13,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:13,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:44:13,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bed6c02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:44:13,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:13,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:13,727 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:13,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56cf5eca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,728 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,728 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:44:13,728 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:13,729 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:44:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:44:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f876bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:44:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:13,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:13,730 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:13,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@396c5f30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,735 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,735 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:44:13,736 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:13,736 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:44:13,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:13,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:44:13,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@404223e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:44:13,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:13,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:13,738 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56644, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38961a7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,739 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40916ac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,739 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:44:13,739 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:13,740 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:44:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:44:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@279bd8ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:44:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:13,741 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:13,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@95f3954, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:13,743 DEBUG [Thread-1095 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:13,743 DEBUG [Thread-1080 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:13,744 DEBUG [Thread-1084 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:13,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:44:13,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:44:13,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:44:13,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:13,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:13,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:13,745 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:13,745 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:13,746 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:13,747 DEBUG [Thread-1089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:44:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:44:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:44:13,748 DEBUG [Thread-1093 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:44:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:13,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:44:13,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:13,750 DEBUG [Thread-1086 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:13,750 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:13,750 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:13,751 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:13,751 DEBUG [Thread-1097 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:13,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:44:13,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:13,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:44:13,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-14T02:44:13,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:44:13,752 DEBUG [Thread-1091 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:13,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T02:44:13,753 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:13,754 DEBUG [Thread-1082 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:13,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:44:13,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:13,754 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:13,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:44:13,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:44:13,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:13,756 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:13,756 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:13,756 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:13,757 DEBUG [Thread-1078 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:44:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:44:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:44:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:13,761 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:13,761 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:13,761 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:13,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:44:13,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:44:13,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:44:13,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:13,766 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:44:13,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 35a73edee503a7eb9039857e3cfc97f1 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T02:44:13,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/4b3a7815863f406d998fee3cb83c6e0f is 50, key is test_row_0/A:col10/1731552253765/Put/seqid=0 2024-11-14T02:44:13,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742055_1231 (size=12001) 2024-11-14T02:44:13,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:13,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552313802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:13,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:13,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:13,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:13,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:13,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552313804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:13,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:13,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:13,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:13,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:13,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552313804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:13,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:13,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:13,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:13,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:13,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50296 deadline: 1731552313806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:13,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:13,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:13,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:13,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:13,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50264 deadline: 1731552313809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:13,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:13,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:13,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:13,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T02:44:13,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T02:44:13,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:13,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:13,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:13,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:13,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:13,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:13,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:13,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552313913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:13,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:13,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:13,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:13,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:13,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552313914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:13,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552313914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:13,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:13,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:13,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50296 deadline: 1731552313924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:13,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:13,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:13,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:13,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:13,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50264 deadline: 1731552313924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:13,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:13,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:13,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:14,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T02:44:14,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:14,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:14,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:14,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T02:44:14,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:14,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552314123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:14,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:14,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:14,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552314123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:14,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:14,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552314124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:14,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:14,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:14,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50296 deadline: 1731552314133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:14,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:14,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50264 deadline: 1731552314133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:14,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:14,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/4b3a7815863f406d998fee3cb83c6e0f 2024-11-14T02:44:14,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T02:44:14,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:14,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:14,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:14,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/eea8ff59bad840298c506ab6f5fd7777 is 50, key is test_row_0/B:col10/1731552253765/Put/seqid=0 2024-11-14T02:44:14,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742056_1232 (size=12001) 2024-11-14T02:44:14,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T02:44:14,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:14,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:14,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:14,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T02:44:14,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:14,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552314433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:14,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:14,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552314433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:14,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:14,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:14,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552314433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:14,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:14,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:14,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:14,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50296 deadline: 1731552314443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:14,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:14,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50264 deadline: 1731552314443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:14,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:14,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:14,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T02:44:14,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:14,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:14,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:14,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/eea8ff59bad840298c506ab6f5fd7777 2024-11-14T02:44:14,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/1c9b5d759ca54d65b8acff3536ae47fb is 50, key is test_row_0/C:col10/1731552253765/Put/seqid=0 2024-11-14T02:44:14,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742057_1233 (size=12001) 2024-11-14T02:44:14,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T02:44:14,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:14,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:14,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:14,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T02:44:14,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:14,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:14,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:14,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T02:44:14,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552314943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552314944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:14,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552314944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:14,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50296 deadline: 1731552314953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:14,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:14,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50264 deadline: 1731552314953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:14,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:14,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T02:44:14,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:14,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:14,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:14,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:15,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/1c9b5d759ca54d65b8acff3536ae47fb 2024-11-14T02:44:15,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/4b3a7815863f406d998fee3cb83c6e0f as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/4b3a7815863f406d998fee3cb83c6e0f 2024-11-14T02:44:15,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/4b3a7815863f406d998fee3cb83c6e0f, entries=150, sequenceid=13, filesize=11.7 K 2024-11-14T02:44:15,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/eea8ff59bad840298c506ab6f5fd7777 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/eea8ff59bad840298c506ab6f5fd7777 2024-11-14T02:44:15,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/eea8ff59bad840298c506ab6f5fd7777, entries=150, sequenceid=13, filesize=11.7 K 2024-11-14T02:44:15,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/1c9b5d759ca54d65b8acff3536ae47fb as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/1c9b5d759ca54d65b8acff3536ae47fb 2024-11-14T02:44:15,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/1c9b5d759ca54d65b8acff3536ae47fb, entries=150, sequenceid=13, filesize=11.7 K 2024-11-14T02:44:15,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 35a73edee503a7eb9039857e3cfc97f1 in 1319ms, sequenceid=13, compaction requested=false 2024-11-14T02:44:15,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:15,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T02:44:15,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:15,134 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 35a73edee503a7eb9039857e3cfc97f1 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-14T02:44:15,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/2ad4f80054d7450e929b5fd33faea73f is 50, key is test_row_0/A:col10/1731552253802/Put/seqid=0 2024-11-14T02:44:15,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742058_1234 (size=12001) 2024-11-14T02:44:15,543 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/2ad4f80054d7450e929b5fd33faea73f 2024-11-14T02:44:15,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/5e9ed33de6844cac8ffa291fe386d47e is 50, key is test_row_0/B:col10/1731552253802/Put/seqid=0 2024-11-14T02:44:15,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742059_1235 (size=12001) 2024-11-14T02:44:15,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T02:44:15,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:15,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:15,955 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/5e9ed33de6844cac8ffa291fe386d47e 2024-11-14T02:44:15,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/d65dd6181b4f44528e77f3d92812fa3a is 50, key is test_row_0/C:col10/1731552253802/Put/seqid=0 2024-11-14T02:44:15,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:15,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552315968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:15,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:15,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:15,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:15,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:15,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552315968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:15,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:15,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:15,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:15,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:15,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552315968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:15,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:15,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:15,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:15,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742060_1236 (size=12001) 2024-11-14T02:44:15,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:15,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:15,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50264 deadline: 1731552315974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:15,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50296 deadline: 1731552315974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:15,977 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_1' in 35a73edee503a7eb9039857e3cfc97f1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:44:15,977 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_2' in 35a73edee503a7eb9039857e3cfc97f1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:44:15,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:15,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:15,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:15,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:15,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:15,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:16,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:16,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552316083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:16,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:16,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552316083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:16,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552316083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:16,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552316293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:16,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:16,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552316294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:16,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:16,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552316294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:16,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:16,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:16,377 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/d65dd6181b4f44528e77f3d92812fa3a 2024-11-14T02:44:16,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/2ad4f80054d7450e929b5fd33faea73f as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/2ad4f80054d7450e929b5fd33faea73f 2024-11-14T02:44:16,462 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/2ad4f80054d7450e929b5fd33faea73f, entries=150, sequenceid=37, filesize=11.7 K 2024-11-14T02:44:16,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/5e9ed33de6844cac8ffa291fe386d47e as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/5e9ed33de6844cac8ffa291fe386d47e 2024-11-14T02:44:16,467 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/5e9ed33de6844cac8ffa291fe386d47e, entries=150, sequenceid=37, filesize=11.7 K 2024-11-14T02:44:16,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/d65dd6181b4f44528e77f3d92812fa3a as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/d65dd6181b4f44528e77f3d92812fa3a 2024-11-14T02:44:16,472 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/d65dd6181b4f44528e77f3d92812fa3a, entries=150, sequenceid=37, filesize=11.7 K 2024-11-14T02:44:16,473 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 35a73edee503a7eb9039857e3cfc97f1 in 1339ms, sequenceid=37, compaction requested=false 2024-11-14T02:44:16,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:16,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:16,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-14T02:44:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-14T02:44:16,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-14T02:44:16,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7190 sec 2024-11-14T02:44:16,478 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.7270 sec 2024-11-14T02:44:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:16,604 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:44:16,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 35a73edee503a7eb9039857e3cfc97f1 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-14T02:44:16,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/69c8792655934d88a91d48450e9cdaec is 50, key is test_row_0/A:col10/1731552256603/Put/seqid=0 2024-11-14T02:44:16,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742061_1237 (size=12001) 2024-11-14T02:44:16,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:16,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552316628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:16,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:16,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:16,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552316630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:16,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552316631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:16,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:16,724 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T02:44:16,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:16,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:16,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552316744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:16,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552316744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:16,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:16,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552316744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:16,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:16,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:16,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:16,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552316953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552316953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:16,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552316953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:16,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:17,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/69c8792655934d88a91d48450e9cdaec 2024-11-14T02:44:17,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/ea978eaced284ac4ac5df231ad0db4bc is 50, key is test_row_0/B:col10/1731552256603/Put/seqid=0 2024-11-14T02:44:17,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742062_1238 (size=12001) 2024-11-14T02:44:17,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/ea978eaced284ac4ac5df231ad0db4bc 2024-11-14T02:44:17,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/4ad332b4aa6b4863884af9cafebceaa0 is 50, key is test_row_0/C:col10/1731552256603/Put/seqid=0 2024-11-14T02:44:17,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742063_1239 (size=12001) 2024-11-14T02:44:17,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:17,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552317263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:17,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552317263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:17,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:17,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552317264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:17,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:17,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/4ad332b4aa6b4863884af9cafebceaa0 2024-11-14T02:44:17,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/69c8792655934d88a91d48450e9cdaec as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/69c8792655934d88a91d48450e9cdaec 2024-11-14T02:44:17,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/69c8792655934d88a91d48450e9cdaec, entries=150, sequenceid=52, filesize=11.7 K 2024-11-14T02:44:17,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/ea978eaced284ac4ac5df231ad0db4bc as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/ea978eaced284ac4ac5df231ad0db4bc 2024-11-14T02:44:17,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/ea978eaced284ac4ac5df231ad0db4bc, entries=150, sequenceid=52, filesize=11.7 K 2024-11-14T02:44:17,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/4ad332b4aa6b4863884af9cafebceaa0 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/4ad332b4aa6b4863884af9cafebceaa0 2024-11-14T02:44:17,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/4ad332b4aa6b4863884af9cafebceaa0, entries=150, sequenceid=52, filesize=11.7 K 2024-11-14T02:44:17,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 35a73edee503a7eb9039857e3cfc97f1 in 868ms, sequenceid=52, compaction requested=true 2024-11-14T02:44:17,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:17,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 35a73edee503a7eb9039857e3cfc97f1:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:44:17,472 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:17,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:17,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 35a73edee503a7eb9039857e3cfc97f1:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:44:17,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:17,472 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:17,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 35a73edee503a7eb9039857e3cfc97f1:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:44:17,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:17,473 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:17,473 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:17,473 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 35a73edee503a7eb9039857e3cfc97f1/A is initiating minor compaction (all files) 2024-11-14T02:44:17,473 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 35a73edee503a7eb9039857e3cfc97f1/B is initiating minor compaction (all files) 2024-11-14T02:44:17,473 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 35a73edee503a7eb9039857e3cfc97f1/B in TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:17,473 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 35a73edee503a7eb9039857e3cfc97f1/A in TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:17,473 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/4b3a7815863f406d998fee3cb83c6e0f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/2ad4f80054d7450e929b5fd33faea73f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/69c8792655934d88a91d48450e9cdaec] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp, totalSize=35.2 K 2024-11-14T02:44:17,473 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/eea8ff59bad840298c506ab6f5fd7777, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/5e9ed33de6844cac8ffa291fe386d47e, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/ea978eaced284ac4ac5df231ad0db4bc] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp, totalSize=35.2 K 2024-11-14T02:44:17,474 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting eea8ff59bad840298c506ab6f5fd7777, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731552253756 2024-11-14T02:44:17,474 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b3a7815863f406d998fee3cb83c6e0f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731552253756 2024-11-14T02:44:17,474 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e9ed33de6844cac8ffa291fe386d47e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731552253802 2024-11-14T02:44:17,474 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ad4f80054d7450e929b5fd33faea73f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731552253802 2024-11-14T02:44:17,474 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting ea978eaced284ac4ac5df231ad0db4bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731552255967 2024-11-14T02:44:17,474 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69c8792655934d88a91d48450e9cdaec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731552255967 2024-11-14T02:44:17,482 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 35a73edee503a7eb9039857e3cfc97f1#A#compaction#198 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:17,482 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 35a73edee503a7eb9039857e3cfc97f1#B#compaction#197 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:17,482 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/64fe35b24da444168effbd44f8fe72a6 is 50, key is test_row_0/A:col10/1731552256603/Put/seqid=0 2024-11-14T02:44:17,482 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/c07541a9038641a08d4a87101fbd7b3a is 50, key is test_row_0/B:col10/1731552256603/Put/seqid=0 2024-11-14T02:44:17,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742064_1240 (size=12104) 2024-11-14T02:44:17,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742065_1241 (size=12104) 2024-11-14T02:44:17,498 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/c07541a9038641a08d4a87101fbd7b3a as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/c07541a9038641a08d4a87101fbd7b3a 2024-11-14T02:44:17,502 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 35a73edee503a7eb9039857e3cfc97f1/B of 35a73edee503a7eb9039857e3cfc97f1 into c07541a9038641a08d4a87101fbd7b3a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:17,502 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:17,502 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., storeName=35a73edee503a7eb9039857e3cfc97f1/B, priority=13, startTime=1731552257472; duration=0sec 2024-11-14T02:44:17,503 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:17,503 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 35a73edee503a7eb9039857e3cfc97f1:B 2024-11-14T02:44:17,503 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:17,504 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:17,504 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 35a73edee503a7eb9039857e3cfc97f1/C is initiating minor compaction (all files) 2024-11-14T02:44:17,504 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 35a73edee503a7eb9039857e3cfc97f1/C in TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:17,504 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/1c9b5d759ca54d65b8acff3536ae47fb, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/d65dd6181b4f44528e77f3d92812fa3a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/4ad332b4aa6b4863884af9cafebceaa0] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp, totalSize=35.2 K 2024-11-14T02:44:17,504 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c9b5d759ca54d65b8acff3536ae47fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731552253756 2024-11-14T02:44:17,505 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting d65dd6181b4f44528e77f3d92812fa3a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731552253802 2024-11-14T02:44:17,505 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ad332b4aa6b4863884af9cafebceaa0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731552255967 2024-11-14T02:44:17,512 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 35a73edee503a7eb9039857e3cfc97f1#C#compaction#199 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:17,512 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/c8254b159900496aa57adf2dc90e6dcd is 50, key is test_row_0/C:col10/1731552256603/Put/seqid=0 2024-11-14T02:44:17,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742066_1242 (size=12104) 2024-11-14T02:44:17,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:17,776 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:44:17,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 35a73edee503a7eb9039857e3cfc97f1 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-14T02:44:17,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/2f5aaced69e848a08b92fe38704ebd00 is 50, key is test_row_0/A:col10/1731552256627/Put/seqid=0 2024-11-14T02:44:17,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742067_1243 (size=16681) 2024-11-14T02:44:17,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:17,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552317789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:17,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:17,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552317789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:17,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:17,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:17,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552317790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:17,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:17,893 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/64fe35b24da444168effbd44f8fe72a6 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/64fe35b24da444168effbd44f8fe72a6 2024-11-14T02:44:17,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:17,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552317894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:17,898 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 35a73edee503a7eb9039857e3cfc97f1/A of 35a73edee503a7eb9039857e3cfc97f1 into 64fe35b24da444168effbd44f8fe72a6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:17,898 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:17,898 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., storeName=35a73edee503a7eb9039857e3cfc97f1/A, priority=13, startTime=1731552257472; duration=0sec 2024-11-14T02:44:17,898 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:17,898 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 35a73edee503a7eb9039857e3cfc97f1:A 2024-11-14T02:44:17,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552317903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:17,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:17,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552317903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:17,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:17,923 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/c8254b159900496aa57adf2dc90e6dcd as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c8254b159900496aa57adf2dc90e6dcd 2024-11-14T02:44:17,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T02:44:17,924 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:17,924 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-14T02:44:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T02:44:17,926 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:17,927 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:17,927 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:17,928 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 35a73edee503a7eb9039857e3cfc97f1/C of 35a73edee503a7eb9039857e3cfc97f1 into c8254b159900496aa57adf2dc90e6dcd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:17,928 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:17,928 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., storeName=35a73edee503a7eb9039857e3cfc97f1/C, priority=13, startTime=1731552257472; duration=0sec 2024-11-14T02:44:17,928 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:17,928 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 35a73edee503a7eb9039857e3cfc97f1:C 2024-11-14T02:44:17,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50264 deadline: 1731552317993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:17,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:17,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50296 deadline: 1731552317993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:17,997 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_2' in 35a73edee503a7eb9039857e3cfc97f1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:44:17,997 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_1' in 35a73edee503a7eb9039857e3cfc97f1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:44:17,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:17,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T02:44:18,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T02:44:18,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:18,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:18,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:18,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:18,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:18,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:18,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:18,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552318103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:18,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:18,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552318113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:18,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:18,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552318113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:18,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:18,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:18,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/2f5aaced69e848a08b92fe38704ebd00 2024-11-14T02:44:18,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/cf9f2d4640d345c681804fd2693eebc0 is 50, key is test_row_0/B:col10/1731552256627/Put/seqid=0 2024-11-14T02:44:18,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742068_1244 (size=12001) 2024-11-14T02:44:18,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T02:44:18,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:18,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:18,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:18,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:18,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T02:44:18,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T02:44:18,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:18,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:18,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:18,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:18,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:18,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:18,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552318413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:18,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:18,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:18,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552318423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:18,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:18,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552318423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:18,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:18,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:18,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T02:44:18,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:18,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:18,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:18,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:18,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:18,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T02:44:18,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/cf9f2d4640d345c681804fd2693eebc0 2024-11-14T02:44:18,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/c61d03b8d91b4b16b12327ca3ef39763 is 50, key is test_row_0/C:col10/1731552256627/Put/seqid=0 2024-11-14T02:44:18,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742069_1245 (size=12001) 2024-11-14T02:44:18,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T02:44:18,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:18,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:18,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:18,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:18,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:18,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T02:44:18,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:18,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:18,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:18,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:18,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:18,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552318923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:18,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:18,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552318933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:18,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:18,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:18,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552318934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:18,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:18,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T02:44:18,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:18,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:18,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:18,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:18,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:19,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/c61d03b8d91b4b16b12327ca3ef39763 2024-11-14T02:44:19,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/2f5aaced69e848a08b92fe38704ebd00 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/2f5aaced69e848a08b92fe38704ebd00 2024-11-14T02:44:19,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/2f5aaced69e848a08b92fe38704ebd00, entries=250, sequenceid=77, filesize=16.3 K 2024-11-14T02:44:19,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/cf9f2d4640d345c681804fd2693eebc0 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/cf9f2d4640d345c681804fd2693eebc0 2024-11-14T02:44:19,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/cf9f2d4640d345c681804fd2693eebc0, entries=150, sequenceid=77, filesize=11.7 K 2024-11-14T02:44:19,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/c61d03b8d91b4b16b12327ca3ef39763 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c61d03b8d91b4b16b12327ca3ef39763 2024-11-14T02:44:19,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c61d03b8d91b4b16b12327ca3ef39763, entries=150, sequenceid=77, filesize=11.7 K 2024-11-14T02:44:19,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 35a73edee503a7eb9039857e3cfc97f1 in 1254ms, sequenceid=77, compaction requested=false 2024-11-14T02:44:19,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T02:44:19,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T02:44:19,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:19,150 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 35a73edee503a7eb9039857e3cfc97f1 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T02:44:19,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/ec6c60e99bcd44b19308c4c632d24a5f is 50, key is test_row_0/A:col10/1731552257785/Put/seqid=0 2024-11-14T02:44:19,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742070_1246 (size=12001) 2024-11-14T02:44:19,570 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/ec6c60e99bcd44b19308c4c632d24a5f 2024-11-14T02:44:19,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/28947feb082a46a5a5f492a98e7e33b9 is 50, key is test_row_0/B:col10/1731552257785/Put/seqid=0 2024-11-14T02:44:19,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742071_1247 (size=12001) 2024-11-14T02:44:19,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:19,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:19,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:19,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552319966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:19,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:19,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:19,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552319969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:19,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:19,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:19,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552319971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:19,996 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/28947feb082a46a5a5f492a98e7e33b9 2024-11-14T02:44:20,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/1359f341464344de84da785b3f2c6747 is 50, key is test_row_0/C:col10/1731552257785/Put/seqid=0 2024-11-14T02:44:20,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742072_1248 (size=12001) 2024-11-14T02:44:20,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T02:44:20,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:20,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552320073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:20,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:20,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:20,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552320083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:20,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:20,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552320084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:20,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:20,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:20,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:20,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552320283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:20,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:20,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:20,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552320293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:20,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552320294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:20,411 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/1359f341464344de84da785b3f2c6747 2024-11-14T02:44:20,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/ec6c60e99bcd44b19308c4c632d24a5f as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/ec6c60e99bcd44b19308c4c632d24a5f 2024-11-14T02:44:20,422 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/ec6c60e99bcd44b19308c4c632d24a5f, entries=150, sequenceid=91, filesize=11.7 K 2024-11-14T02:44:20,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/28947feb082a46a5a5f492a98e7e33b9 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/28947feb082a46a5a5f492a98e7e33b9 2024-11-14T02:44:20,427 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/28947feb082a46a5a5f492a98e7e33b9, entries=150, sequenceid=91, filesize=11.7 K 2024-11-14T02:44:20,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/1359f341464344de84da785b3f2c6747 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/1359f341464344de84da785b3f2c6747 2024-11-14T02:44:20,432 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/1359f341464344de84da785b3f2c6747, entries=150, sequenceid=91, filesize=11.7 K 2024-11-14T02:44:20,433 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 35a73edee503a7eb9039857e3cfc97f1 in 1283ms, sequenceid=91, compaction requested=true 2024-11-14T02:44:20,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:20,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:20,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-14T02:44:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-14T02:44:20,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-14T02:44:20,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5070 sec 2024-11-14T02:44:20,438 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.5130 sec 2024-11-14T02:44:20,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:20,596 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:44:20,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 35a73edee503a7eb9039857e3cfc97f1 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T02:44:20,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/8df6be96a30242eca9c2e6d10d273b1a is 50, key is test_row_0/A:col10/1731552259956/Put/seqid=0 2024-11-14T02:44:20,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742073_1249 (size=14341) 2024-11-14T02:44:20,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:20,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552320610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:20,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:20,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552320611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:20,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:20,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:20,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:20,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552320615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:20,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:20,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:20,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:20,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552320723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:20,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552320723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:20,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552320723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:20,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552320934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:20,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:20,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552320934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:20,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:20,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552320934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:21,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/8df6be96a30242eca9c2e6d10d273b1a 2024-11-14T02:44:21,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/87557d0c53fb4ddba1512d31c5cb8d33 is 50, key is test_row_0/B:col10/1731552259956/Put/seqid=0 2024-11-14T02:44:21,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742074_1250 (size=12001) 2024-11-14T02:44:21,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:21,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552321243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:21,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:21,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552321243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:21,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552321243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:21,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:21,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/87557d0c53fb4ddba1512d31c5cb8d33 2024-11-14T02:44:21,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/275f48f0405845738b627e7a2949a464 is 50, key is test_row_0/C:col10/1731552259956/Put/seqid=0 2024-11-14T02:44:21,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742075_1251 (size=12001) 2024-11-14T02:44:21,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/275f48f0405845738b627e7a2949a464 2024-11-14T02:44:21,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/8df6be96a30242eca9c2e6d10d273b1a as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/8df6be96a30242eca9c2e6d10d273b1a 2024-11-14T02:44:21,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/8df6be96a30242eca9c2e6d10d273b1a, entries=200, sequenceid=116, filesize=14.0 K 2024-11-14T02:44:21,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/87557d0c53fb4ddba1512d31c5cb8d33 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/87557d0c53fb4ddba1512d31c5cb8d33 2024-11-14T02:44:21,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/87557d0c53fb4ddba1512d31c5cb8d33, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T02:44:21,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/275f48f0405845738b627e7a2949a464 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/275f48f0405845738b627e7a2949a464 2024-11-14T02:44:21,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/275f48f0405845738b627e7a2949a464, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T02:44:21,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 35a73edee503a7eb9039857e3cfc97f1 in 876ms, sequenceid=116, compaction requested=true 2024-11-14T02:44:21,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:21,472 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:44:21,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 35a73edee503a7eb9039857e3cfc97f1:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:44:21,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:21,473 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:44:21,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 35a73edee503a7eb9039857e3cfc97f1:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:44:21,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:21,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 35a73edee503a7eb9039857e3cfc97f1:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:44:21,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:21,474 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55127 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:44:21,474 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 35a73edee503a7eb9039857e3cfc97f1/A is initiating minor compaction (all files) 2024-11-14T02:44:21,474 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 35a73edee503a7eb9039857e3cfc97f1/A in TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:21,474 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/64fe35b24da444168effbd44f8fe72a6, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/2f5aaced69e848a08b92fe38704ebd00, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/ec6c60e99bcd44b19308c4c632d24a5f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/8df6be96a30242eca9c2e6d10d273b1a] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp, totalSize=53.8 K 2024-11-14T02:44:21,476 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:44:21,476 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 35a73edee503a7eb9039857e3cfc97f1/B is initiating minor compaction (all files) 2024-11-14T02:44:21,476 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 35a73edee503a7eb9039857e3cfc97f1/B in TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:21,476 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/c07541a9038641a08d4a87101fbd7b3a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/cf9f2d4640d345c681804fd2693eebc0, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/28947feb082a46a5a5f492a98e7e33b9, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/87557d0c53fb4ddba1512d31c5cb8d33] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp, totalSize=47.0 K 2024-11-14T02:44:21,476 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64fe35b24da444168effbd44f8fe72a6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731552255967 2024-11-14T02:44:21,477 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting c07541a9038641a08d4a87101fbd7b3a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731552255967 2024-11-14T02:44:21,478 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f5aaced69e848a08b92fe38704ebd00, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731552256627 2024-11-14T02:44:21,478 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting cf9f2d4640d345c681804fd2693eebc0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731552256627 2024-11-14T02:44:21,478 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec6c60e99bcd44b19308c4c632d24a5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731552257785 2024-11-14T02:44:21,479 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 28947feb082a46a5a5f492a98e7e33b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731552257785 2024-11-14T02:44:21,479 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8df6be96a30242eca9c2e6d10d273b1a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731552259956 2024-11-14T02:44:21,479 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 87557d0c53fb4ddba1512d31c5cb8d33, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731552259956 2024-11-14T02:44:21,494 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 35a73edee503a7eb9039857e3cfc97f1#B#compaction#209 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:21,494 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/b7d14de88129413a9f92641a18cc1923 is 50, key is test_row_0/B:col10/1731552259956/Put/seqid=0 2024-11-14T02:44:21,505 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 35a73edee503a7eb9039857e3cfc97f1#A#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:21,506 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/fd413c2308cc4dc6a21936275be9357e is 50, key is test_row_0/A:col10/1731552259956/Put/seqid=0 2024-11-14T02:44:21,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742076_1252 (size=12241) 2024-11-14T02:44:21,539 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/b7d14de88129413a9f92641a18cc1923 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/b7d14de88129413a9f92641a18cc1923 2024-11-14T02:44:21,546 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 35a73edee503a7eb9039857e3cfc97f1/B of 35a73edee503a7eb9039857e3cfc97f1 into b7d14de88129413a9f92641a18cc1923(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:21,546 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:21,546 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., storeName=35a73edee503a7eb9039857e3cfc97f1/B, priority=12, startTime=1731552261473; duration=0sec 2024-11-14T02:44:21,546 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:21,546 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 35a73edee503a7eb9039857e3cfc97f1:B 2024-11-14T02:44:21,546 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:44:21,547 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:44:21,548 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 35a73edee503a7eb9039857e3cfc97f1/C is initiating minor compaction (all files) 2024-11-14T02:44:21,548 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 35a73edee503a7eb9039857e3cfc97f1/C in TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:21,548 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c8254b159900496aa57adf2dc90e6dcd, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c61d03b8d91b4b16b12327ca3ef39763, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/1359f341464344de84da785b3f2c6747, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/275f48f0405845738b627e7a2949a464] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp, totalSize=47.0 K 2024-11-14T02:44:21,549 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting c8254b159900496aa57adf2dc90e6dcd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731552255967 2024-11-14T02:44:21,549 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting c61d03b8d91b4b16b12327ca3ef39763, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731552256627 2024-11-14T02:44:21,550 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 1359f341464344de84da785b3f2c6747, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731552257785 2024-11-14T02:44:21,550 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 275f48f0405845738b627e7a2949a464, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731552259956 2024-11-14T02:44:21,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742077_1253 (size=12241) 2024-11-14T02:44:21,563 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/fd413c2308cc4dc6a21936275be9357e as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/fd413c2308cc4dc6a21936275be9357e 2024-11-14T02:44:21,570 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 35a73edee503a7eb9039857e3cfc97f1/A of 35a73edee503a7eb9039857e3cfc97f1 into fd413c2308cc4dc6a21936275be9357e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:21,570 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:21,570 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., storeName=35a73edee503a7eb9039857e3cfc97f1/A, priority=12, startTime=1731552261472; duration=0sec 2024-11-14T02:44:21,570 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 35a73edee503a7eb9039857e3cfc97f1#C#compaction#211 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:21,570 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:21,570 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 35a73edee503a7eb9039857e3cfc97f1:A 2024-11-14T02:44:21,570 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/79831d676e40413b9b1a4ded2438ad0f is 50, key is test_row_0/C:col10/1731552259956/Put/seqid=0 2024-11-14T02:44:21,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742078_1254 (size=12241) 2024-11-14T02:44:21,584 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/79831d676e40413b9b1a4ded2438ad0f as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/79831d676e40413b9b1a4ded2438ad0f 2024-11-14T02:44:21,592 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 35a73edee503a7eb9039857e3cfc97f1/C of 35a73edee503a7eb9039857e3cfc97f1 into 79831d676e40413b9b1a4ded2438ad0f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:21,592 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:21,592 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., storeName=35a73edee503a7eb9039857e3cfc97f1/C, priority=12, startTime=1731552261473; duration=0sec 2024-11-14T02:44:21,592 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:21,592 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 35a73edee503a7eb9039857e3cfc97f1:C 2024-11-14T02:44:21,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:21,755 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:44:21,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 35a73edee503a7eb9039857e3cfc97f1 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-14T02:44:21,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/06e8d02a1a164c55b871bd5a6023a738 is 50, key is test_row_0/A:col10/1731552261755/Put/seqid=0 2024-11-14T02:44:21,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742079_1255 (size=12101) 2024-11-14T02:44:21,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/06e8d02a1a164c55b871bd5a6023a738 2024-11-14T02:44:21,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/ed909a2e648c46cd8cfaac3ee9dfd50b is 50, key is test_row_0/B:col10/1731552261755/Put/seqid=0 2024-11-14T02:44:21,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:21,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552321781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:21,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552321781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:21,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552321782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:21,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742080_1256 (size=12101) 2024-11-14T02:44:21,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552321893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:21,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552321893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552321893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:22,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50264 deadline: 1731552322013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:22,016 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_1' in 35a73edee503a7eb9039857e3cfc97f1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:44:22,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:22,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50296 deadline: 1731552322023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:22,026 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_2' in 35a73edee503a7eb9039857e3cfc97f1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:44:22,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T02:44:22,083 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:22,083 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:22,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-14T02:44:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T02:44:22,085 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:22,086 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:22,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:22,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552322103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:22,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:22,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552322103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:22,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:22,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552322103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:22,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T02:44:22,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/ed909a2e648c46cd8cfaac3ee9dfd50b 2024-11-14T02:44:22,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/8866d91ff0c64dafbdb6f82a70a98a21 is 50, key is test_row_0/C:col10/1731552261755/Put/seqid=0 2024-11-14T02:44:22,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742081_1257 (size=12101) 2024-11-14T02:44:22,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T02:44:22,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:22,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:22,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:22,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:22,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:22,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T02:44:22,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:22,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:22,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:22,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:22,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T02:44:22,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552322413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:22,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:22,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552322413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:22,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:22,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:22,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552322413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:22,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:22,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T02:44:22,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:22,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:22,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:22,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:22,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:22,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/8866d91ff0c64dafbdb6f82a70a98a21 2024-11-14T02:44:22,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/06e8d02a1a164c55b871bd5a6023a738 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/06e8d02a1a164c55b871bd5a6023a738 2024-11-14T02:44:22,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/06e8d02a1a164c55b871bd5a6023a738, entries=150, sequenceid=133, filesize=11.8 K 2024-11-14T02:44:22,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/ed909a2e648c46cd8cfaac3ee9dfd50b as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/ed909a2e648c46cd8cfaac3ee9dfd50b 2024-11-14T02:44:22,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/ed909a2e648c46cd8cfaac3ee9dfd50b, entries=150, sequenceid=133, filesize=11.8 K 2024-11-14T02:44:22,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/8866d91ff0c64dafbdb6f82a70a98a21 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/8866d91ff0c64dafbdb6f82a70a98a21 2024-11-14T02:44:22,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/8866d91ff0c64dafbdb6f82a70a98a21, entries=150, sequenceid=133, filesize=11.8 K 2024-11-14T02:44:22,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 35a73edee503a7eb9039857e3cfc97f1 in 869ms, sequenceid=133, compaction requested=false 2024-11-14T02:44:22,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:22,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T02:44:22,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:22,696 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 35a73edee503a7eb9039857e3cfc97f1 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-14T02:44:22,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/b8aa9ee8571646c494a1a75c2c774f0b is 50, key is test_row_0/A:col10/1731552261781/Put/seqid=0 2024-11-14T02:44:22,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742082_1258 (size=12151) 2024-11-14T02:44:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T02:44:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:22,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:22,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552322936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:22,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552322938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:22,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:22,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552322940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:22,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:23,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552323043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:23,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:23,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552323053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552323053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:23,106 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/b8aa9ee8571646c494a1a75c2c774f0b 2024-11-14T02:44:23,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/76096481c8df41cc89fa38f12b40762e is 50, key is test_row_0/B:col10/1731552261781/Put/seqid=0 2024-11-14T02:44:23,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742083_1259 (size=12151) 2024-11-14T02:44:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T02:44:23,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552323253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:23,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:23,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552323263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552323263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:23,528 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/76096481c8df41cc89fa38f12b40762e 2024-11-14T02:44:23,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/c8d9b49a0afd4b0486c51ff9bb8b1cc9 is 50, key is test_row_0/C:col10/1731552261781/Put/seqid=0 2024-11-14T02:44:23,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742084_1260 (size=12151) 2024-11-14T02:44:23,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552323563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:23,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:23,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552323573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:23,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:23,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552323573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:23,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:23,939 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/c8d9b49a0afd4b0486c51ff9bb8b1cc9 2024-11-14T02:44:23,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/b8aa9ee8571646c494a1a75c2c774f0b as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/b8aa9ee8571646c494a1a75c2c774f0b 2024-11-14T02:44:23,946 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/b8aa9ee8571646c494a1a75c2c774f0b, entries=150, sequenceid=155, filesize=11.9 K 2024-11-14T02:44:23,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/76096481c8df41cc89fa38f12b40762e as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/76096481c8df41cc89fa38f12b40762e 2024-11-14T02:44:23,950 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/76096481c8df41cc89fa38f12b40762e, entries=150, sequenceid=155, filesize=11.9 K 2024-11-14T02:44:23,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/c8d9b49a0afd4b0486c51ff9bb8b1cc9 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c8d9b49a0afd4b0486c51ff9bb8b1cc9 2024-11-14T02:44:23,954 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c8d9b49a0afd4b0486c51ff9bb8b1cc9, entries=150, sequenceid=155, filesize=11.9 K 2024-11-14T02:44:23,955 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 35a73edee503a7eb9039857e3cfc97f1 in 1260ms, sequenceid=155, compaction requested=true 2024-11-14T02:44:23,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:23,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:23,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-14T02:44:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-14T02:44:23,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-14T02:44:23,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8700 sec 2024-11-14T02:44:23,960 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.8760 sec 2024-11-14T02:44:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:24,076 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:44:24,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 35a73edee503a7eb9039857e3cfc97f1 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-14T02:44:24,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/c05d9389d8ca4f5e8ef85c43386f2c24 is 50, key is test_row_0/A:col10/1731552264075/Put/seqid=0 2024-11-14T02:44:24,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742085_1261 (size=14541) 2024-11-14T02:44:24,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:24,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552324134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552324135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:24,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:24,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:24,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:24,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552324138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:24,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T02:44:24,234 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:24,234 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-14T02:44:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T02:44:24,236 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:24,236 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:24,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:24,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:24,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552324243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:24,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:24,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:24,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552324243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:24,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:24,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:24,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552324243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:24,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:24,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T02:44:24,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T02:44:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:24,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:24,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:24,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552324454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:24,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:24,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:24,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552324454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552324454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:24,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:24,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:24,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/c05d9389d8ca4f5e8ef85c43386f2c24 2024-11-14T02:44:24,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/e833ee1baf2f416688359f6df8321d24 is 50, key is test_row_0/B:col10/1731552264075/Put/seqid=0 2024-11-14T02:44:24,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742086_1262 (size=12151) 2024-11-14T02:44:24,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/e833ee1baf2f416688359f6df8321d24 2024-11-14T02:44:24,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/00ffb42b7c7f4e74a3b2ab091a3a2995 is 50, key is test_row_0/C:col10/1731552264075/Put/seqid=0 2024-11-14T02:44:24,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742087_1263 (size=12151) 2024-11-14T02:44:24,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T02:44:24,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:24,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:24,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:24,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:24,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T02:44:24,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T02:44:24,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:24,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:24,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:24,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:24,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:24,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552324763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:24,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:24,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552324763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:24,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:24,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:24,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552324763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:24,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T02:44:24,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:24,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:24,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:24,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:24,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T02:44:24,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/00ffb42b7c7f4e74a3b2ab091a3a2995 2024-11-14T02:44:24,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/c05d9389d8ca4f5e8ef85c43386f2c24 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/c05d9389d8ca4f5e8ef85c43386f2c24 2024-11-14T02:44:24,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/c05d9389d8ca4f5e8ef85c43386f2c24, entries=200, sequenceid=170, filesize=14.2 K 2024-11-14T02:44:24,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/e833ee1baf2f416688359f6df8321d24 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/e833ee1baf2f416688359f6df8321d24 2024-11-14T02:44:24,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/e833ee1baf2f416688359f6df8321d24, entries=150, sequenceid=170, filesize=11.9 K 2024-11-14T02:44:24,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/00ffb42b7c7f4e74a3b2ab091a3a2995 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/00ffb42b7c7f4e74a3b2ab091a3a2995 2024-11-14T02:44:24,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/00ffb42b7c7f4e74a3b2ab091a3a2995, entries=150, sequenceid=170, filesize=11.9 K 2024-11-14T02:44:24,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 35a73edee503a7eb9039857e3cfc97f1 in 847ms, sequenceid=170, compaction requested=true 2024-11-14T02:44:24,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:24,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 35a73edee503a7eb9039857e3cfc97f1:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:44:24,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:24,924 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:44:24,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 35a73edee503a7eb9039857e3cfc97f1:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:44:24,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:24,924 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:44:24,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 35a73edee503a7eb9039857e3cfc97f1:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:44:24,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:24,925 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:44:24,925 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51034 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:44:24,925 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 35a73edee503a7eb9039857e3cfc97f1/B is initiating minor compaction (all files) 2024-11-14T02:44:24,925 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 35a73edee503a7eb9039857e3cfc97f1/A is initiating minor compaction (all files) 2024-11-14T02:44:24,925 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 35a73edee503a7eb9039857e3cfc97f1/B in TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:24,925 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 35a73edee503a7eb9039857e3cfc97f1/A in TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:24,925 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/b7d14de88129413a9f92641a18cc1923, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/ed909a2e648c46cd8cfaac3ee9dfd50b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/76096481c8df41cc89fa38f12b40762e, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/e833ee1baf2f416688359f6df8321d24] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp, totalSize=47.5 K 2024-11-14T02:44:24,925 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/fd413c2308cc4dc6a21936275be9357e, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/06e8d02a1a164c55b871bd5a6023a738, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/b8aa9ee8571646c494a1a75c2c774f0b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/c05d9389d8ca4f5e8ef85c43386f2c24] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp, totalSize=49.8 K 2024-11-14T02:44:24,926 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting b7d14de88129413a9f92641a18cc1923, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731552259956 2024-11-14T02:44:24,926 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd413c2308cc4dc6a21936275be9357e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731552259956 2024-11-14T02:44:24,926 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06e8d02a1a164c55b871bd5a6023a738, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731552260608 2024-11-14T02:44:24,926 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting ed909a2e648c46cd8cfaac3ee9dfd50b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731552260608 2024-11-14T02:44:24,926 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8aa9ee8571646c494a1a75c2c774f0b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731552261773 2024-11-14T02:44:24,926 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 76096481c8df41cc89fa38f12b40762e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731552261773 2024-11-14T02:44:24,927 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting c05d9389d8ca4f5e8ef85c43386f2c24, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731552262926 2024-11-14T02:44:24,927 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting e833ee1baf2f416688359f6df8321d24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731552262926 2024-11-14T02:44:24,936 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 35a73edee503a7eb9039857e3cfc97f1#B#compaction#221 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:24,936 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/96810c57edbc4292a44a225b60c77886 is 50, key is test_row_0/B:col10/1731552264075/Put/seqid=0 2024-11-14T02:44:24,936 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 35a73edee503a7eb9039857e3cfc97f1#A#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:24,937 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/fa200a6b09c24fa5b65fcf9e3572d689 is 50, key is test_row_0/A:col10/1731552264075/Put/seqid=0 2024-11-14T02:44:24,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742088_1264 (size=12527) 2024-11-14T02:44:24,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742089_1265 (size=12527) 2024-11-14T02:44:24,962 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/96810c57edbc4292a44a225b60c77886 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/96810c57edbc4292a44a225b60c77886 2024-11-14T02:44:24,969 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 35a73edee503a7eb9039857e3cfc97f1/B of 35a73edee503a7eb9039857e3cfc97f1 into 96810c57edbc4292a44a225b60c77886(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:24,969 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:24,969 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., storeName=35a73edee503a7eb9039857e3cfc97f1/B, priority=12, startTime=1731552264924; duration=0sec 2024-11-14T02:44:24,969 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:24,969 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 35a73edee503a7eb9039857e3cfc97f1:B 2024-11-14T02:44:24,969 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:44:24,972 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/fa200a6b09c24fa5b65fcf9e3572d689 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/fa200a6b09c24fa5b65fcf9e3572d689 2024-11-14T02:44:24,972 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:44:24,972 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 35a73edee503a7eb9039857e3cfc97f1/C is initiating minor compaction (all files) 2024-11-14T02:44:24,972 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 35a73edee503a7eb9039857e3cfc97f1/C in TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:24,972 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/79831d676e40413b9b1a4ded2438ad0f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/8866d91ff0c64dafbdb6f82a70a98a21, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c8d9b49a0afd4b0486c51ff9bb8b1cc9, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/00ffb42b7c7f4e74a3b2ab091a3a2995] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp, totalSize=47.5 K 2024-11-14T02:44:24,972 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 79831d676e40413b9b1a4ded2438ad0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731552259956 2024-11-14T02:44:24,973 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 8866d91ff0c64dafbdb6f82a70a98a21, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731552260608 2024-11-14T02:44:24,973 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting c8d9b49a0afd4b0486c51ff9bb8b1cc9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731552261773 2024-11-14T02:44:24,973 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 00ffb42b7c7f4e74a3b2ab091a3a2995, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731552262926 2024-11-14T02:44:24,976 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 35a73edee503a7eb9039857e3cfc97f1/A of 35a73edee503a7eb9039857e3cfc97f1 into fa200a6b09c24fa5b65fcf9e3572d689(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:24,976 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:24,976 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., storeName=35a73edee503a7eb9039857e3cfc97f1/A, priority=12, startTime=1731552264924; duration=0sec 2024-11-14T02:44:24,976 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:24,976 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 35a73edee503a7eb9039857e3cfc97f1:A 2024-11-14T02:44:24,981 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 35a73edee503a7eb9039857e3cfc97f1#C#compaction#223 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:24,981 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/c668375d881a4bfbb5c7593b8eb575c2 is 50, key is test_row_0/C:col10/1731552264075/Put/seqid=0 2024-11-14T02:44:24,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742090_1266 (size=12527) 2024-11-14T02:44:24,990 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/c668375d881a4bfbb5c7593b8eb575c2 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c668375d881a4bfbb5c7593b8eb575c2 2024-11-14T02:44:24,995 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 35a73edee503a7eb9039857e3cfc97f1/C of 35a73edee503a7eb9039857e3cfc97f1 into c668375d881a4bfbb5c7593b8eb575c2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:24,995 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:24,995 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., storeName=35a73edee503a7eb9039857e3cfc97f1/C, priority=12, startTime=1731552264924; duration=0sec 2024-11-14T02:44:24,995 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:24,995 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 35a73edee503a7eb9039857e3cfc97f1:C 2024-11-14T02:44:24,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T02:44:25,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:25,000 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 35a73edee503a7eb9039857e3cfc97f1 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-14T02:44:25,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/fa8fc185752447c69e93517de923a7bf is 50, key is test_row_0/A:col10/1731552264122/Put/seqid=0 2024-11-14T02:44:25,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742091_1267 (size=12151) 2024-11-14T02:44:25,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:25,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:25,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:25,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552325285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:25,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:25,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:25,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552325287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:25,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:25,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552325288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:25,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:25,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T02:44:25,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:25,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552325394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:25,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552325394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:25,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:25,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552325394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:25,409 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/fa8fc185752447c69e93517de923a7bf 2024-11-14T02:44:25,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/af34d8a814a14e9cad8b96b313bdac8f is 50, key is test_row_0/B:col10/1731552264122/Put/seqid=0 2024-11-14T02:44:25,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742092_1268 (size=12151) 2024-11-14T02:44:25,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:25,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552325604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:25,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:25,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:25,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552325604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:25,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:25,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:25,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552325604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:25,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:25,819 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/af34d8a814a14e9cad8b96b313bdac8f 2024-11-14T02:44:25,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/84db99a6c13346e1b20bd96818f0eb43 is 50, key is test_row_0/C:col10/1731552264122/Put/seqid=0 2024-11-14T02:44:25,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742093_1269 (size=12151) 2024-11-14T02:44:25,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:25,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552325913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:25,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552325913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:25,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552325913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:26,237 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/84db99a6c13346e1b20bd96818f0eb43 2024-11-14T02:44:26,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/fa8fc185752447c69e93517de923a7bf as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/fa8fc185752447c69e93517de923a7bf 2024-11-14T02:44:26,244 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/fa8fc185752447c69e93517de923a7bf, entries=150, sequenceid=194, filesize=11.9 K 2024-11-14T02:44:26,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/af34d8a814a14e9cad8b96b313bdac8f as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/af34d8a814a14e9cad8b96b313bdac8f 2024-11-14T02:44:26,249 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/af34d8a814a14e9cad8b96b313bdac8f, entries=150, sequenceid=194, filesize=11.9 K 2024-11-14T02:44:26,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/84db99a6c13346e1b20bd96818f0eb43 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/84db99a6c13346e1b20bd96818f0eb43 2024-11-14T02:44:26,253 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/84db99a6c13346e1b20bd96818f0eb43, entries=150, sequenceid=194, filesize=11.9 K 2024-11-14T02:44:26,254 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 35a73edee503a7eb9039857e3cfc97f1 in 1254ms, sequenceid=194, compaction requested=false 2024-11-14T02:44:26,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:26,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:26,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-14T02:44:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-14T02:44:26,258 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-14T02:44:26,258 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0190 sec 2024-11-14T02:44:26,259 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.0240 sec 2024-11-14T02:44:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T02:44:26,384 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:26,384 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-14T02:44:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T02:44:26,386 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:26,386 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:26,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:26,425 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:44:26,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 35a73edee503a7eb9039857e3cfc97f1 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-14T02:44:26,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:26,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552326449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552326450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:26,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552326450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:26,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/e4f842ffe48140319dff2003be6b3d51 is 50, key is test_row_0/A:col10/1731552266424/Put/seqid=0 2024-11-14T02:44:26,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742094_1270 (size=14541) 2024-11-14T02:44:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T02:44:26,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T02:44:26,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:26,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:26,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:26,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:26,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:26,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552326564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:26,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552326564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:26,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:26,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:26,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:26,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:26,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:26,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:26,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552326564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:26,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:26,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:26,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:26,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T02:44:26,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:26,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:26,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:26,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:26,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T02:44:26,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:26,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552326773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:26,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:26,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:26,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:26,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:26,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552326773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:26,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:26,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:26,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:26,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:26,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552326774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:26,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:26,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:26,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:26,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T02:44:26,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:26,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:26,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:26,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:26,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:26,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/e4f842ffe48140319dff2003be6b3d51 2024-11-14T02:44:26,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/70142c65140449dca0d33d37e387dd4a is 50, key is test_row_0/B:col10/1731552266424/Put/seqid=0 2024-11-14T02:44:26,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742095_1271 (size=12151) 2024-11-14T02:44:26,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T02:44:26,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:26,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:26,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:26,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:26,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T02:44:27,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552327083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:27,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:27,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552327083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:27,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:27,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:27,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552327083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:27,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:27,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:27,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T02:44:27,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:27,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:27,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:27,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:27,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:27,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/70142c65140449dca0d33d37e387dd4a 2024-11-14T02:44:27,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/84569c3101964c1b9696f2efdc35ed96 is 50, key is test_row_0/C:col10/1731552266424/Put/seqid=0 2024-11-14T02:44:27,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742096_1272 (size=12151) 2024-11-14T02:44:27,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T02:44:27,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:27,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:27,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:27,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:27,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:27,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T02:44:27,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:27,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:27,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:27,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:27,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T02:44:27,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552327593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552327593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:27,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552327593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:27,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T02:44:27,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:27,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:27,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:27,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:27,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:27,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/84569c3101964c1b9696f2efdc35ed96 2024-11-14T02:44:27,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/e4f842ffe48140319dff2003be6b3d51 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/e4f842ffe48140319dff2003be6b3d51 2024-11-14T02:44:27,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/e4f842ffe48140319dff2003be6b3d51, entries=200, sequenceid=212, filesize=14.2 K 2024-11-14T02:44:27,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/70142c65140449dca0d33d37e387dd4a as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/70142c65140449dca0d33d37e387dd4a 2024-11-14T02:44:27,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/70142c65140449dca0d33d37e387dd4a, entries=150, sequenceid=212, filesize=11.9 K 2024-11-14T02:44:27,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/84569c3101964c1b9696f2efdc35ed96 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/84569c3101964c1b9696f2efdc35ed96 2024-11-14T02:44:27,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/84569c3101964c1b9696f2efdc35ed96, entries=150, sequenceid=212, filesize=11.9 K 2024-11-14T02:44:27,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 35a73edee503a7eb9039857e3cfc97f1 in 1280ms, sequenceid=212, compaction requested=true 2024-11-14T02:44:27,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:27,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 35a73edee503a7eb9039857e3cfc97f1:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:44:27,706 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:27,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:27,706 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:27,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 35a73edee503a7eb9039857e3cfc97f1:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:44:27,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:27,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 35a73edee503a7eb9039857e3cfc97f1:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:44:27,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:27,708 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:27,708 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 35a73edee503a7eb9039857e3cfc97f1/B is initiating minor compaction (all files) 2024-11-14T02:44:27,708 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 35a73edee503a7eb9039857e3cfc97f1/B in TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:27,708 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/96810c57edbc4292a44a225b60c77886, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/af34d8a814a14e9cad8b96b313bdac8f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/70142c65140449dca0d33d37e387dd4a] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp, totalSize=36.0 K 2024-11-14T02:44:27,708 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:27,708 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 35a73edee503a7eb9039857e3cfc97f1/A is initiating minor compaction (all files) 2024-11-14T02:44:27,708 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 35a73edee503a7eb9039857e3cfc97f1/A in TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:27,708 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/fa200a6b09c24fa5b65fcf9e3572d689, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/fa8fc185752447c69e93517de923a7bf, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/e4f842ffe48140319dff2003be6b3d51] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp, totalSize=38.3 K 2024-11-14T02:44:27,708 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 96810c57edbc4292a44a225b60c77886, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731552262926 2024-11-14T02:44:27,708 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting af34d8a814a14e9cad8b96b313bdac8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731552264122 2024-11-14T02:44:27,708 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa200a6b09c24fa5b65fcf9e3572d689, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731552262926 2024-11-14T02:44:27,709 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 70142c65140449dca0d33d37e387dd4a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731552266424 2024-11-14T02:44:27,709 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa8fc185752447c69e93517de923a7bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731552264122 2024-11-14T02:44:27,709 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4f842ffe48140319dff2003be6b3d51, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731552265287 2024-11-14T02:44:27,718 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 35a73edee503a7eb9039857e3cfc97f1#B#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:27,718 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/6992d725bbe5430eb80afeeb7086b195 is 50, key is test_row_0/B:col10/1731552266424/Put/seqid=0 2024-11-14T02:44:27,721 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 35a73edee503a7eb9039857e3cfc97f1#A#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:27,721 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/5b0b8f86f44c4b18a17d0516cf50a046 is 50, key is test_row_0/A:col10/1731552266424/Put/seqid=0 2024-11-14T02:44:27,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742097_1273 (size=12629) 2024-11-14T02:44:27,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742098_1274 (size=12629) 2024-11-14T02:44:27,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T02:44:27,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:27,761 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 35a73edee503a7eb9039857e3cfc97f1 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-14T02:44:27,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/8e4972e7d972404da887ac26deee82e5 is 50, key is test_row_0/A:col10/1731552266447/Put/seqid=0 2024-11-14T02:44:27,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742099_1275 (size=12151) 2024-11-14T02:44:27,769 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/8e4972e7d972404da887ac26deee82e5 2024-11-14T02:44:27,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/d9855367b1734580b11ee9dc8d6bae1d is 50, key is test_row_0/B:col10/1731552266447/Put/seqid=0 2024-11-14T02:44:27,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742100_1276 (size=12151) 2024-11-14T02:44:28,146 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/6992d725bbe5430eb80afeeb7086b195 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/6992d725bbe5430eb80afeeb7086b195 2024-11-14T02:44:28,146 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/5b0b8f86f44c4b18a17d0516cf50a046 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/5b0b8f86f44c4b18a17d0516cf50a046 2024-11-14T02:44:28,151 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 35a73edee503a7eb9039857e3cfc97f1/A of 35a73edee503a7eb9039857e3cfc97f1 into 5b0b8f86f44c4b18a17d0516cf50a046(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:28,151 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 35a73edee503a7eb9039857e3cfc97f1/B of 35a73edee503a7eb9039857e3cfc97f1 into 6992d725bbe5430eb80afeeb7086b195(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:28,151 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:28,151 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:28,151 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., storeName=35a73edee503a7eb9039857e3cfc97f1/A, priority=13, startTime=1731552267706; duration=0sec 2024-11-14T02:44:28,151 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., storeName=35a73edee503a7eb9039857e3cfc97f1/B, priority=13, startTime=1731552267706; duration=0sec 2024-11-14T02:44:28,151 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:28,151 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:28,151 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 35a73edee503a7eb9039857e3cfc97f1:A 2024-11-14T02:44:28,151 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 35a73edee503a7eb9039857e3cfc97f1:B 2024-11-14T02:44:28,152 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:28,153 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:28,153 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 35a73edee503a7eb9039857e3cfc97f1/C is initiating minor compaction (all files) 2024-11-14T02:44:28,153 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 35a73edee503a7eb9039857e3cfc97f1/C in TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:28,153 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c668375d881a4bfbb5c7593b8eb575c2, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/84db99a6c13346e1b20bd96818f0eb43, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/84569c3101964c1b9696f2efdc35ed96] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp, totalSize=36.0 K 2024-11-14T02:44:28,154 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting c668375d881a4bfbb5c7593b8eb575c2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731552262926 2024-11-14T02:44:28,154 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84db99a6c13346e1b20bd96818f0eb43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731552264122 2024-11-14T02:44:28,154 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84569c3101964c1b9696f2efdc35ed96, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731552266424 2024-11-14T02:44:28,161 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 35a73edee503a7eb9039857e3cfc97f1#C#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:28,161 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/246c2f31348849978670d51270cf11b1 is 50, key is test_row_0/C:col10/1731552266424/Put/seqid=0 2024-11-14T02:44:28,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742101_1277 (size=12629) 2024-11-14T02:44:28,181 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/d9855367b1734580b11ee9dc8d6bae1d 2024-11-14T02:44:28,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/544060562cd943db92542c043deba15b is 50, key is test_row_0/C:col10/1731552266447/Put/seqid=0 2024-11-14T02:44:28,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742102_1278 (size=12151) 2024-11-14T02:44:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T02:44:28,569 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/246c2f31348849978670d51270cf11b1 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/246c2f31348849978670d51270cf11b1 2024-11-14T02:44:28,573 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 35a73edee503a7eb9039857e3cfc97f1/C of 35a73edee503a7eb9039857e3cfc97f1 into 246c2f31348849978670d51270cf11b1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:28,573 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:28,573 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., storeName=35a73edee503a7eb9039857e3cfc97f1/C, priority=13, startTime=1731552267706; duration=0sec 2024-11-14T02:44:28,573 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:28,573 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 35a73edee503a7eb9039857e3cfc97f1:C 2024-11-14T02:44:28,591 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/544060562cd943db92542c043deba15b 2024-11-14T02:44:28,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/8e4972e7d972404da887ac26deee82e5 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/8e4972e7d972404da887ac26deee82e5 2024-11-14T02:44:28,598 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/8e4972e7d972404da887ac26deee82e5, entries=150, sequenceid=230, filesize=11.9 K 2024-11-14T02:44:28,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/d9855367b1734580b11ee9dc8d6bae1d as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/d9855367b1734580b11ee9dc8d6bae1d 2024-11-14T02:44:28,602 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/d9855367b1734580b11ee9dc8d6bae1d, entries=150, sequenceid=230, filesize=11.9 K 2024-11-14T02:44:28,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/544060562cd943db92542c043deba15b as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/544060562cd943db92542c043deba15b 2024-11-14T02:44:28,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:28,607 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/544060562cd943db92542c043deba15b, entries=150, sequenceid=230, filesize=11.9 K 2024-11-14T02:44:28,608 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=13.42 KB/13740 for 35a73edee503a7eb9039857e3cfc97f1 in 847ms, sequenceid=230, compaction requested=false 2024-11-14T02:44:28,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:28,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:28,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-14T02:44:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-14T02:44:28,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-14T02:44:28,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2220 sec 2024-11-14T02:44:28,613 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.2270 sec 2024-11-14T02:44:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:28,617 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:44:28,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 35a73edee503a7eb9039857e3cfc97f1 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T02:44:28,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/7b66c03b38de426fae42abe6051b89cc is 50, key is test_row_0/A:col10/1731552268613/Put/seqid=0 2024-11-14T02:44:28,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742103_1279 (size=16931) 2024-11-14T02:44:28,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552328649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552328650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:28,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552328653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:28,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:28,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552328763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:28,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:28,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552328763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:28,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552328763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:28,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:28,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:28,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:28,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552328973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:28,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:28,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:28,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552328973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:28,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:28,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552328973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:28,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:28,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:29,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/7b66c03b38de426fae42abe6051b89cc 2024-11-14T02:44:29,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/3ae1e8c646d24600ae4f55ed94cef86e is 50, key is test_row_0/B:col10/1731552268613/Put/seqid=0 2024-11-14T02:44:29,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742104_1280 (size=12151) 2024-11-14T02:44:29,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/3ae1e8c646d24600ae4f55ed94cef86e 2024-11-14T02:44:29,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/22295686b7724613949c6fdd4f710b6b is 50, key is test_row_0/C:col10/1731552268613/Put/seqid=0 2024-11-14T02:44:29,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742105_1281 (size=12151) 2024-11-14T02:44:29,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552329283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:29,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:29,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:29,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:29,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552329284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:29,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:29,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:29,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:29,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552329284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:29,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:29,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:29,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:29,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/22295686b7724613949c6fdd4f710b6b 2024-11-14T02:44:29,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/7b66c03b38de426fae42abe6051b89cc as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/7b66c03b38de426fae42abe6051b89cc 2024-11-14T02:44:29,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/7b66c03b38de426fae42abe6051b89cc, entries=250, sequenceid=245, filesize=16.5 K 2024-11-14T02:44:29,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/3ae1e8c646d24600ae4f55ed94cef86e as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/3ae1e8c646d24600ae4f55ed94cef86e 2024-11-14T02:44:29,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/3ae1e8c646d24600ae4f55ed94cef86e, entries=150, sequenceid=245, filesize=11.9 K 2024-11-14T02:44:29,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/22295686b7724613949c6fdd4f710b6b as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/22295686b7724613949c6fdd4f710b6b 2024-11-14T02:44:29,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/22295686b7724613949c6fdd4f710b6b, entries=150, sequenceid=245, filesize=11.9 K 2024-11-14T02:44:29,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 35a73edee503a7eb9039857e3cfc97f1 in 865ms, sequenceid=245, compaction requested=true 2024-11-14T02:44:29,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:29,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 35a73edee503a7eb9039857e3cfc97f1:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:44:29,482 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:29,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:29,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 35a73edee503a7eb9039857e3cfc97f1:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:44:29,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:29,483 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:29,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 35a73edee503a7eb9039857e3cfc97f1:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:44:29,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:29,483 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:29,483 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:29,483 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 35a73edee503a7eb9039857e3cfc97f1/B is initiating minor compaction (all files) 2024-11-14T02:44:29,483 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 35a73edee503a7eb9039857e3cfc97f1/A is initiating minor compaction (all files) 2024-11-14T02:44:29,483 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 35a73edee503a7eb9039857e3cfc97f1/B in TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:29,483 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 35a73edee503a7eb9039857e3cfc97f1/A in TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:29,484 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/6992d725bbe5430eb80afeeb7086b195, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/d9855367b1734580b11ee9dc8d6bae1d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/3ae1e8c646d24600ae4f55ed94cef86e] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp, totalSize=36.1 K 2024-11-14T02:44:29,484 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/5b0b8f86f44c4b18a17d0516cf50a046, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/8e4972e7d972404da887ac26deee82e5, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/7b66c03b38de426fae42abe6051b89cc] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp, totalSize=40.7 K 2024-11-14T02:44:29,484 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 6992d725bbe5430eb80afeeb7086b195, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731552266424 2024-11-14T02:44:29,484 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b0b8f86f44c4b18a17d0516cf50a046, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731552266424 2024-11-14T02:44:29,484 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e4972e7d972404da887ac26deee82e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731552266447 2024-11-14T02:44:29,484 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting d9855367b1734580b11ee9dc8d6bae1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731552266447 2024-11-14T02:44:29,485 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b66c03b38de426fae42abe6051b89cc, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731552268607 2024-11-14T02:44:29,485 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ae1e8c646d24600ae4f55ed94cef86e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731552268607 2024-11-14T02:44:29,492 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 35a73edee503a7eb9039857e3cfc97f1#B#compaction#239 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:29,493 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/fca40485f2e54918b972ba20ba31f2ab is 50, key is test_row_0/B:col10/1731552268613/Put/seqid=0 2024-11-14T02:44:29,505 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 35a73edee503a7eb9039857e3cfc97f1#A#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:29,506 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/07ef0837cd2044558af5d7c8a6c78a07 is 50, key is test_row_0/A:col10/1731552268613/Put/seqid=0 2024-11-14T02:44:29,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742106_1282 (size=12731) 2024-11-14T02:44:29,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742107_1283 (size=12731) 2024-11-14T02:44:29,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:29,795 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:44:29,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 35a73edee503a7eb9039857e3cfc97f1 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T02:44:29,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/b3151c2ce5754fff82366b4033ddf1bf is 50, key is test_row_0/A:col10/1731552268645/Put/seqid=0 2024-11-14T02:44:29,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742108_1284 (size=14741) 2024-11-14T02:44:29,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552329804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:29,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:29,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:29,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:29,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:29,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552329806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:29,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:29,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:29,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:29,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:29,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552329807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:29,913 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/fca40485f2e54918b972ba20ba31f2ab as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/fca40485f2e54918b972ba20ba31f2ab 2024-11-14T02:44:29,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:29,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552329913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:29,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:29,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552329913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:29,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:29,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:29,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:29,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:29,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:29,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:29,918 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/07ef0837cd2044558af5d7c8a6c78a07 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/07ef0837cd2044558af5d7c8a6c78a07 2024-11-14T02:44:29,918 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 35a73edee503a7eb9039857e3cfc97f1/B of 35a73edee503a7eb9039857e3cfc97f1 into fca40485f2e54918b972ba20ba31f2ab(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:29,918 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:29,918 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., storeName=35a73edee503a7eb9039857e3cfc97f1/B, priority=13, startTime=1731552269482; duration=0sec 2024-11-14T02:44:29,918 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:29,918 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 35a73edee503a7eb9039857e3cfc97f1:B 2024-11-14T02:44:29,918 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:29,919 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:29,919 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 35a73edee503a7eb9039857e3cfc97f1/C is initiating minor compaction (all files) 2024-11-14T02:44:29,919 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 35a73edee503a7eb9039857e3cfc97f1/C in TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:29,919 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/246c2f31348849978670d51270cf11b1, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/544060562cd943db92542c043deba15b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/22295686b7724613949c6fdd4f710b6b] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp, totalSize=36.1 K 2024-11-14T02:44:29,920 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 246c2f31348849978670d51270cf11b1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731552266424 2024-11-14T02:44:29,920 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 544060562cd943db92542c043deba15b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731552266447 2024-11-14T02:44:29,920 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 22295686b7724613949c6fdd4f710b6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731552268607 2024-11-14T02:44:29,923 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 35a73edee503a7eb9039857e3cfc97f1/A of 35a73edee503a7eb9039857e3cfc97f1 into 07ef0837cd2044558af5d7c8a6c78a07(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:29,923 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:29,923 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., storeName=35a73edee503a7eb9039857e3cfc97f1/A, priority=13, startTime=1731552269482; duration=0sec 2024-11-14T02:44:29,923 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:29,923 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 35a73edee503a7eb9039857e3cfc97f1:A 2024-11-14T02:44:29,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:29,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552329923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:29,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:29,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:29,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:29,927 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 35a73edee503a7eb9039857e3cfc97f1#C#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:29,928 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/847947259e154cf099d72adfec4c38fb is 50, key is test_row_0/C:col10/1731552268613/Put/seqid=0 2024-11-14T02:44:29,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742109_1285 (size=12731) 2024-11-14T02:44:29,950 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/847947259e154cf099d72adfec4c38fb as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/847947259e154cf099d72adfec4c38fb 2024-11-14T02:44:29,955 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 35a73edee503a7eb9039857e3cfc97f1/C of 35a73edee503a7eb9039857e3cfc97f1 into 847947259e154cf099d72adfec4c38fb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:29,955 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:29,955 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., storeName=35a73edee503a7eb9039857e3cfc97f1/C, priority=13, startTime=1731552269483; duration=0sec 2024-11-14T02:44:29,955 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:29,955 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 35a73edee503a7eb9039857e3cfc97f1:C 2024-11-14T02:44:30,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:30,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:30,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552330124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:30,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552330124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:30,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:30,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:30,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:30,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:30,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:30,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:30,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:30,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552330133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:30,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:30,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:30,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/b3151c2ce5754fff82366b4033ddf1bf 2024-11-14T02:44:30,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/4d4b5f2e03834c3d8ceb1d1959f13832 is 50, key is test_row_0/B:col10/1731552268645/Put/seqid=0 2024-11-14T02:44:30,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742110_1286 (size=12301) 2024-11-14T02:44:30,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:30,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552330433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:30,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:30,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:30,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:30,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:30,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552330433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:30,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:30,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:30,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:30,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552330443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:30,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T02:44:30,544 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:30,544 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:30,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-14T02:44:30,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T02:44:30,545 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:30,546 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:30,546 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:30,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/4d4b5f2e03834c3d8ceb1d1959f13832 2024-11-14T02:44:30,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/c59485d288844618b2af73e9723078c0 is 50, key is test_row_0/C:col10/1731552268645/Put/seqid=0 2024-11-14T02:44:30,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742111_1287 (size=12301) 2024-11-14T02:44:30,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T02:44:30,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-14T02:44:30,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:30,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:30,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:30,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:30,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:30,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:30,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-14T02:44:30,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:30,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:30,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:30,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:30,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:30,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T02:44:30,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552330944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:30,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:30,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:30,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:30,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552330944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:30,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:30,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:30,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:30,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552330953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:31,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-14T02:44:31,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:31,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:31,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:31,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:31,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:31,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/c59485d288844618b2af73e9723078c0 2024-11-14T02:44:31,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/b3151c2ce5754fff82366b4033ddf1bf as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/b3151c2ce5754fff82366b4033ddf1bf 2024-11-14T02:44:31,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/b3151c2ce5754fff82366b4033ddf1bf, entries=200, sequenceid=270, filesize=14.4 K 2024-11-14T02:44:31,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/4d4b5f2e03834c3d8ceb1d1959f13832 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/4d4b5f2e03834c3d8ceb1d1959f13832 2024-11-14T02:44:31,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/4d4b5f2e03834c3d8ceb1d1959f13832, entries=150, sequenceid=270, filesize=12.0 K 2024-11-14T02:44:31,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/c59485d288844618b2af73e9723078c0 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c59485d288844618b2af73e9723078c0 2024-11-14T02:44:31,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c59485d288844618b2af73e9723078c0, entries=150, sequenceid=270, filesize=12.0 K 2024-11-14T02:44:31,043 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 35a73edee503a7eb9039857e3cfc97f1 in 1248ms, sequenceid=270, compaction requested=false 2024-11-14T02:44:31,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:31,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-14T02:44:31,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:31,161 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 35a73edee503a7eb9039857e3cfc97f1 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T02:44:31,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/ac6c16ede1b34ae2a73922435451c173 is 50, key is test_row_0/A:col10/1731552269806/Put/seqid=0 2024-11-14T02:44:31,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742112_1288 (size=12301) 2024-11-14T02:44:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T02:44:31,568 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/ac6c16ede1b34ae2a73922435451c173 2024-11-14T02:44:31,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/0cd9b65de0f842abbf87c8db6096e702 is 50, key is test_row_0/B:col10/1731552269806/Put/seqid=0 2024-11-14T02:44:31,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742113_1289 (size=12301) 2024-11-14T02:44:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T02:44:31,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:31,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:31,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:31,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552331988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:31,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:31,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:31,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:31,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:31,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552331990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:31,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:31,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:31,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:31,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:31,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552331991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:31,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:31,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:31,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:32,006 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/0cd9b65de0f842abbf87c8db6096e702 2024-11-14T02:44:32,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/81da659d582a4cd9aa28d5e6e562894d is 50, key is test_row_0/C:col10/1731552269806/Put/seqid=0 2024-11-14T02:44:32,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742114_1290 (size=12301) 2024-11-14T02:44:32,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:32,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50264 deadline: 1731552332033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:32,035 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_1' in 35a73edee503a7eb9039857e3cfc97f1 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:44:32,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:32,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:32,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50296 deadline: 1731552332084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:32,086 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_2' in 35a73edee503a7eb9039857e3cfc97f1 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18280 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:44:32,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:32,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:32,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552332093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:32,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:32,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:32,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552332103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:32,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:32,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:32,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552332103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:32,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:32,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:32,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552332304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:32,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:32,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552332313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:32,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:32,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552332313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:32,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:32,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:32,415 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/81da659d582a4cd9aa28d5e6e562894d 2024-11-14T02:44:32,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/ac6c16ede1b34ae2a73922435451c173 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/ac6c16ede1b34ae2a73922435451c173 2024-11-14T02:44:32,422 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/ac6c16ede1b34ae2a73922435451c173, entries=150, sequenceid=284, filesize=12.0 K 2024-11-14T02:44:32,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/0cd9b65de0f842abbf87c8db6096e702 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/0cd9b65de0f842abbf87c8db6096e702 2024-11-14T02:44:32,426 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/0cd9b65de0f842abbf87c8db6096e702, entries=150, sequenceid=284, filesize=12.0 K 2024-11-14T02:44:32,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/81da659d582a4cd9aa28d5e6e562894d as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/81da659d582a4cd9aa28d5e6e562894d 2024-11-14T02:44:32,431 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/81da659d582a4cd9aa28d5e6e562894d, entries=150, sequenceid=284, filesize=12.0 K 2024-11-14T02:44:32,432 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 35a73edee503a7eb9039857e3cfc97f1 in 1270ms, sequenceid=284, compaction requested=true 2024-11-14T02:44:32,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:32,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:32,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-14T02:44:32,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-14T02:44:32,436 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-14T02:44:32,436 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8870 sec 2024-11-14T02:44:32,438 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.8920 sec 2024-11-14T02:44:32,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:32,616 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:44:32,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 35a73edee503a7eb9039857e3cfc97f1 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-14T02:44:32,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/e0306a004263431c95a794387e6185dc is 50, key is test_row_0/A:col10/1731552271989/Put/seqid=0 2024-11-14T02:44:32,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742115_1291 (size=14741) 2024-11-14T02:44:32,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:32,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552332628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:32,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:32,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552332630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:32,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:32,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552332632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:32,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:32,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T02:44:32,693 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:32,694 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:32,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-14T02:44:32,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-14T02:44:32,696 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:32,696 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:32,696 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:32,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:32,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552332743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:32,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:32,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552332744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:32,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552332744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:32,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:32,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-14T02:44:32,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-14T02:44:32,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:32,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:32,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:32,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:32,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:32,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:32,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:32,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552332954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:32,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:32,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:32,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552332954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:32,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:32,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:32,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552332954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:32,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:32,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:33,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-14T02:44:33,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:33,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:33,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:33,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:33,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:33,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-14T02:44:33,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/e0306a004263431c95a794387e6185dc 2024-11-14T02:44:33,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/7e4a78bf09f04c7db8681ec5d8ca6d15 is 50, key is test_row_0/B:col10/1731552271989/Put/seqid=0 2024-11-14T02:44:33,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742116_1292 (size=12301) 2024-11-14T02:44:33,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-14T02:44:33,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:33,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:33,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:33,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:33,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:33,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:33,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:33,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552333263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:33,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:33,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:33,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:33,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:33,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552333263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:33,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:33,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:33,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:33,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:33,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552333263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:33,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:33,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:33,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:33,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-14T02:44:33,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:33,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:33,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:33,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:33,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:33,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:33,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-14T02:44:33,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/7e4a78bf09f04c7db8681ec5d8ca6d15 2024-11-14T02:44:33,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/2e14fc56237e422eb53adc561dff37f6 is 50, key is test_row_0/C:col10/1731552271989/Put/seqid=0 2024-11-14T02:44:33,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742117_1293 (size=12301) 2024-11-14T02:44:33,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-14T02:44:33,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:33,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:33,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:33,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:33,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:33,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:33,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-14T02:44:33,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:33,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:33,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:33,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:33,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:33,755 INFO [Thread-1091 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1091. 2024-11-14T02:44:33,755 DEBUG [Thread-1091 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:33,755 DEBUG [Thread-1091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:33,755 DEBUG [Thread-1091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:33,755 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:33,755 INFO [Thread-1095 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1095. 2024-11-14T02:44:33,755 DEBUG [Thread-1095 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:33,755 DEBUG [Thread-1095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:33,755 DEBUG [Thread-1095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:33,755 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:33,755 INFO [Thread-1097 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1097. 2024-11-14T02:44:33,755 DEBUG [Thread-1097 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:33,755 DEBUG [Thread-1097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:33,756 DEBUG [Thread-1097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:33,756 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:33,758 INFO [Thread-1093 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1093. 2024-11-14T02:44:33,758 DEBUG [Thread-1093 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:33,758 DEBUG [Thread-1093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:33,758 DEBUG [Thread-1093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:33,758 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:33,758 INFO [Thread-1089 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1089. 2024-11-14T02:44:33,759 DEBUG [Thread-1089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:33,759 DEBUG [Thread-1089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:33,759 DEBUG [Thread-1089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:33,759 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:33,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-14T02:44:33,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:33,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:33,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:33,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:33,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:33,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:33,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:33,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:33,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:33,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50324 deadline: 1731552333773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:33,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50354 deadline: 1731552333773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:33,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50348 deadline: 1731552333773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:33,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-14T02:44:33,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/2e14fc56237e422eb53adc561dff37f6 2024-11-14T02:44:33,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/e0306a004263431c95a794387e6185dc as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/e0306a004263431c95a794387e6185dc 2024-11-14T02:44:33,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/e0306a004263431c95a794387e6185dc, entries=200, sequenceid=310, filesize=14.4 K 2024-11-14T02:44:33,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/7e4a78bf09f04c7db8681ec5d8ca6d15 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/7e4a78bf09f04c7db8681ec5d8ca6d15 2024-11-14T02:44:33,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/7e4a78bf09f04c7db8681ec5d8ca6d15, entries=150, sequenceid=310, filesize=12.0 K 2024-11-14T02:44:33,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/2e14fc56237e422eb53adc561dff37f6 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/2e14fc56237e422eb53adc561dff37f6 2024-11-14T02:44:33,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/2e14fc56237e422eb53adc561dff37f6, entries=150, sequenceid=310, filesize=12.0 K 2024-11-14T02:44:33,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 35a73edee503a7eb9039857e3cfc97f1 in 1265ms, sequenceid=310, compaction requested=true 2024-11-14T02:44:33,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:33,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 35a73edee503a7eb9039857e3cfc97f1:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:44:33,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:33,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 35a73edee503a7eb9039857e3cfc97f1:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:44:33,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:33,881 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:44:33,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 35a73edee503a7eb9039857e3cfc97f1:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:44:33,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:33,881 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:44:33,882 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54514 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:44:33,882 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:44:33,882 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 35a73edee503a7eb9039857e3cfc97f1/A is initiating minor compaction (all files) 2024-11-14T02:44:33,882 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 35a73edee503a7eb9039857e3cfc97f1/B is initiating minor compaction (all files) 2024-11-14T02:44:33,882 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 35a73edee503a7eb9039857e3cfc97f1/A in TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:33,882 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 35a73edee503a7eb9039857e3cfc97f1/B in TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:33,882 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/07ef0837cd2044558af5d7c8a6c78a07, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/b3151c2ce5754fff82366b4033ddf1bf, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/ac6c16ede1b34ae2a73922435451c173, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/e0306a004263431c95a794387e6185dc] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp, totalSize=53.2 K 2024-11-14T02:44:33,882 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/fca40485f2e54918b972ba20ba31f2ab, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/4d4b5f2e03834c3d8ceb1d1959f13832, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/0cd9b65de0f842abbf87c8db6096e702, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/7e4a78bf09f04c7db8681ec5d8ca6d15] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp, totalSize=48.5 K 2024-11-14T02:44:33,883 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting fca40485f2e54918b972ba20ba31f2ab, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731552268607 2024-11-14T02:44:33,883 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07ef0837cd2044558af5d7c8a6c78a07, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731552268607 2024-11-14T02:44:33,883 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3151c2ce5754fff82366b4033ddf1bf, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731552268645 2024-11-14T02:44:33,883 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d4b5f2e03834c3d8ceb1d1959f13832, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731552268645 2024-11-14T02:44:33,883 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cd9b65de0f842abbf87c8db6096e702, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731552269796 2024-11-14T02:44:33,883 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac6c16ede1b34ae2a73922435451c173, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731552269796 2024-11-14T02:44:33,883 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e4a78bf09f04c7db8681ec5d8ca6d15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731552271989 2024-11-14T02:44:33,884 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0306a004263431c95a794387e6185dc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731552271983 2024-11-14T02:44:33,891 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 35a73edee503a7eb9039857e3cfc97f1#B#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:33,892 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 35a73edee503a7eb9039857e3cfc97f1#A#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:33,892 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/c461d68aec9844f4bc489ff0960aae10 is 50, key is test_row_0/B:col10/1731552271989/Put/seqid=0 2024-11-14T02:44:33,892 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/2a8e93fbfc384a29be5c733be3661bca is 50, key is test_row_0/A:col10/1731552271989/Put/seqid=0 2024-11-14T02:44:33,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742119_1295 (size=13017) 2024-11-14T02:44:33,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742118_1294 (size=13017) 2024-11-14T02:44:33,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-14T02:44:33,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:33,918 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 35a73edee503a7eb9039857e3cfc97f1 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-14T02:44:33,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/83aa231be600441885674e1ff254cbc5 is 50, key is test_row_0/A:col10/1731552272627/Put/seqid=0 2024-11-14T02:44:33,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742120_1296 (size=12301) 2024-11-14T02:44:34,307 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/2a8e93fbfc384a29be5c733be3661bca as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/2a8e93fbfc384a29be5c733be3661bca 2024-11-14T02:44:34,307 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/c461d68aec9844f4bc489ff0960aae10 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/c461d68aec9844f4bc489ff0960aae10 2024-11-14T02:44:34,313 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 35a73edee503a7eb9039857e3cfc97f1/B of 35a73edee503a7eb9039857e3cfc97f1 into c461d68aec9844f4bc489ff0960aae10(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:34,313 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 35a73edee503a7eb9039857e3cfc97f1/A of 35a73edee503a7eb9039857e3cfc97f1 into 2a8e93fbfc384a29be5c733be3661bca(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:34,313 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:34,313 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:34,313 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., storeName=35a73edee503a7eb9039857e3cfc97f1/B, priority=12, startTime=1731552273881; duration=0sec 2024-11-14T02:44:34,313 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., storeName=35a73edee503a7eb9039857e3cfc97f1/A, priority=12, startTime=1731552273881; duration=0sec 2024-11-14T02:44:34,313 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:34,313 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 35a73edee503a7eb9039857e3cfc97f1:A 2024-11-14T02:44:34,313 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:34,313 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 35a73edee503a7eb9039857e3cfc97f1:B 2024-11-14T02:44:34,313 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:44:34,315 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:44:34,315 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 35a73edee503a7eb9039857e3cfc97f1/C is initiating minor compaction (all files) 2024-11-14T02:44:34,315 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 35a73edee503a7eb9039857e3cfc97f1/C in TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:34,315 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/847947259e154cf099d72adfec4c38fb, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c59485d288844618b2af73e9723078c0, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/81da659d582a4cd9aa28d5e6e562894d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/2e14fc56237e422eb53adc561dff37f6] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp, totalSize=48.5 K 2024-11-14T02:44:34,315 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 847947259e154cf099d72adfec4c38fb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731552268607 2024-11-14T02:44:34,316 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting c59485d288844618b2af73e9723078c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731552268645 2024-11-14T02:44:34,316 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81da659d582a4cd9aa28d5e6e562894d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731552269796 2024-11-14T02:44:34,316 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e14fc56237e422eb53adc561dff37f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731552271989 2024-11-14T02:44:34,323 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 35a73edee503a7eb9039857e3cfc97f1#C#compaction#254 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:34,323 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/2c0efec743ab4491a3016740c4d4bc0a is 50, key is test_row_0/C:col10/1731552271989/Put/seqid=0 2024-11-14T02:44:34,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742121_1297 (size=13017) 2024-11-14T02:44:34,329 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/83aa231be600441885674e1ff254cbc5 2024-11-14T02:44:34,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/8d001d7d7af54e77aeb8066826af85d5 is 50, key is test_row_0/B:col10/1731552272627/Put/seqid=0 2024-11-14T02:44:34,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742122_1298 (size=12301) 2024-11-14T02:44:34,740 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/8d001d7d7af54e77aeb8066826af85d5 2024-11-14T02:44:34,741 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/2c0efec743ab4491a3016740c4d4bc0a as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/2c0efec743ab4491a3016740c4d4bc0a 2024-11-14T02:44:34,745 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 35a73edee503a7eb9039857e3cfc97f1/C of 35a73edee503a7eb9039857e3cfc97f1 into 2c0efec743ab4491a3016740c4d4bc0a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:34,745 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:34,745 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., storeName=35a73edee503a7eb9039857e3cfc97f1/C, priority=12, startTime=1731552273881; duration=0sec 2024-11-14T02:44:34,745 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:34,745 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 35a73edee503a7eb9039857e3cfc97f1:C 2024-11-14T02:44:34,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/bf7454df70e24d76acf8335b56306708 is 50, key is test_row_0/C:col10/1731552272627/Put/seqid=0 2024-11-14T02:44:34,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742123_1299 (size=12301) 2024-11-14T02:44:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:34,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. as already flushing 2024-11-14T02:44:34,785 INFO [Thread-1082 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1082. 2024-11-14T02:44:34,785 INFO [Thread-1086 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1086. 2024-11-14T02:44:34,785 INFO [Thread-1078 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1078. 2024-11-14T02:44:34,785 DEBUG [Thread-1082 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:34,785 DEBUG [Thread-1086 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:34,785 DEBUG [Thread-1078 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:34,785 DEBUG [Thread-1082 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:34,785 DEBUG [Thread-1086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:34,785 DEBUG [Thread-1078 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:34,786 DEBUG [Thread-1078 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:34,786 DEBUG [Thread-1082 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:34,786 DEBUG [Thread-1086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:34,786 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:34,786 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:34,786 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-14T02:44:35,152 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/bf7454df70e24d76acf8335b56306708 2024-11-14T02:44:35,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/83aa231be600441885674e1ff254cbc5 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/83aa231be600441885674e1ff254cbc5 2024-11-14T02:44:35,169 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/83aa231be600441885674e1ff254cbc5, entries=150, sequenceid=320, filesize=12.0 K 2024-11-14T02:44:35,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/8d001d7d7af54e77aeb8066826af85d5 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/8d001d7d7af54e77aeb8066826af85d5 2024-11-14T02:44:35,174 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/8d001d7d7af54e77aeb8066826af85d5, entries=150, sequenceid=320, filesize=12.0 K 2024-11-14T02:44:35,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/bf7454df70e24d76acf8335b56306708 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/bf7454df70e24d76acf8335b56306708 2024-11-14T02:44:35,180 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/bf7454df70e24d76acf8335b56306708, entries=150, sequenceid=320, filesize=12.0 K 2024-11-14T02:44:35,181 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=20.13 KB/20610 for 35a73edee503a7eb9039857e3cfc97f1 in 1263ms, sequenceid=320, compaction requested=false 2024-11-14T02:44:35,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 35a73edee503a7eb9039857e3cfc97f1: 2024-11-14T02:44:35,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:35,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-14T02:44:35,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-14T02:44:35,184 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-14T02:44:35,184 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4860 sec 2024-11-14T02:44:35,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 2.4910 sec 2024-11-14T02:44:36,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-14T02:44:36,855 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:39,750 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T02:44:42,058 INFO [Thread-1084 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1084. 2024-11-14T02:44:42,058 DEBUG [Thread-1084 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:42,058 DEBUG [Thread-1084 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:42,059 DEBUG [Thread-1084 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:42,059 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:42,128 INFO [Thread-1080 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1080. 2024-11-14T02:44:42,128 DEBUG [Thread-1080 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:42,129 DEBUG [Thread-1080 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:42,129 DEBUG [Thread-1080 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:42,129 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:42,129 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T02:44:42,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-14T02:44:42,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-11-14T02:44:42,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-14T02:44:42,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-14T02:44:42,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-14T02:44:42,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T02:44:42,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4495 2024-11-14T02:44:42,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4448 2024-11-14T02:44:42,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4399 2024-11-14T02:44:42,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4499 2024-11-14T02:44:42,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4440 2024-11-14T02:44:42,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T02:44:42,130 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T02:44:42,130 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T02:44:42,130 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:42,131 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:42,131 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:42,132 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T02:44:42,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:42,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-14T02:44:42,136 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552282136"}]},"ts":"1731552282136"} 2024-11-14T02:44:42,138 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T02:44:42,138 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T02:44:42,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T02:44:42,141 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=35a73edee503a7eb9039857e3cfc97f1, UNASSIGN}] 2024-11-14T02:44:42,142 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=35a73edee503a7eb9039857e3cfc97f1, UNASSIGN 2024-11-14T02:44:42,143 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=35a73edee503a7eb9039857e3cfc97f1, regionState=CLOSING, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:42,144 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=35a73edee503a7eb9039857e3cfc97f1, UNASSIGN because future has completed 2024-11-14T02:44:42,145 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T02:44:42,145 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259}] 2024-11-14T02:44:42,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-14T02:44:42,299 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:42,299 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T02:44:42,299 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing 35a73edee503a7eb9039857e3cfc97f1, disabling compactions & flushes 2024-11-14T02:44:42,299 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:42,299 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:42,299 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. after waiting 0 ms 2024-11-14T02:44:42,299 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:42,300 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(2902): Flushing 35a73edee503a7eb9039857e3cfc97f1 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-14T02:44:42,309 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/c903eccf647f4a478e3a87ba0337e4c2 is 50, key is test_row_0/A:col10/1731552274784/Put/seqid=0 2024-11-14T02:44:42,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742124_1300 (size=12301) 2024-11-14T02:44:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-14T02:44:42,715 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/c903eccf647f4a478e3a87ba0337e4c2 2024-11-14T02:44:42,729 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/f1885a39dca04c4190283846e667fc51 is 50, key is test_row_0/B:col10/1731552274784/Put/seqid=0 2024-11-14T02:44:42,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742125_1301 (size=12301) 2024-11-14T02:44:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-14T02:44:43,133 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/f1885a39dca04c4190283846e667fc51 2024-11-14T02:44:43,139 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/67e8d3f2920d41119f52dace84ccdabf is 50, key is test_row_0/C:col10/1731552274784/Put/seqid=0 2024-11-14T02:44:43,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742126_1302 (size=12301) 2024-11-14T02:44:43,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-14T02:44:43,545 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/67e8d3f2920d41119f52dace84ccdabf 2024-11-14T02:44:43,585 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/A/c903eccf647f4a478e3a87ba0337e4c2 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/c903eccf647f4a478e3a87ba0337e4c2 2024-11-14T02:44:43,592 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/c903eccf647f4a478e3a87ba0337e4c2, entries=150, sequenceid=331, filesize=12.0 K 2024-11-14T02:44:43,594 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/B/f1885a39dca04c4190283846e667fc51 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/f1885a39dca04c4190283846e667fc51 2024-11-14T02:44:43,599 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/f1885a39dca04c4190283846e667fc51, entries=150, sequenceid=331, filesize=12.0 K 2024-11-14T02:44:43,600 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/.tmp/C/67e8d3f2920d41119f52dace84ccdabf as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/67e8d3f2920d41119f52dace84ccdabf 2024-11-14T02:44:43,605 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/67e8d3f2920d41119f52dace84ccdabf, entries=150, sequenceid=331, filesize=12.0 K 2024-11-14T02:44:43,606 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 35a73edee503a7eb9039857e3cfc97f1 in 1307ms, sequenceid=331, compaction requested=true 2024-11-14T02:44:43,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/4b3a7815863f406d998fee3cb83c6e0f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/2ad4f80054d7450e929b5fd33faea73f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/64fe35b24da444168effbd44f8fe72a6, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/69c8792655934d88a91d48450e9cdaec, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/2f5aaced69e848a08b92fe38704ebd00, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/ec6c60e99bcd44b19308c4c632d24a5f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/8df6be96a30242eca9c2e6d10d273b1a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/fd413c2308cc4dc6a21936275be9357e, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/06e8d02a1a164c55b871bd5a6023a738, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/b8aa9ee8571646c494a1a75c2c774f0b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/c05d9389d8ca4f5e8ef85c43386f2c24, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/fa200a6b09c24fa5b65fcf9e3572d689, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/fa8fc185752447c69e93517de923a7bf, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/e4f842ffe48140319dff2003be6b3d51, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/5b0b8f86f44c4b18a17d0516cf50a046, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/8e4972e7d972404da887ac26deee82e5, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/7b66c03b38de426fae42abe6051b89cc, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/07ef0837cd2044558af5d7c8a6c78a07, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/b3151c2ce5754fff82366b4033ddf1bf, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/ac6c16ede1b34ae2a73922435451c173, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/e0306a004263431c95a794387e6185dc] to archive 2024-11-14T02:44:43,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:44:43,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/4b3a7815863f406d998fee3cb83c6e0f to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/4b3a7815863f406d998fee3cb83c6e0f 2024-11-14T02:44:43,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/2ad4f80054d7450e929b5fd33faea73f to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/2ad4f80054d7450e929b5fd33faea73f 2024-11-14T02:44:43,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/64fe35b24da444168effbd44f8fe72a6 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/64fe35b24da444168effbd44f8fe72a6 2024-11-14T02:44:43,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/69c8792655934d88a91d48450e9cdaec to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/69c8792655934d88a91d48450e9cdaec 2024-11-14T02:44:43,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/2f5aaced69e848a08b92fe38704ebd00 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/2f5aaced69e848a08b92fe38704ebd00 2024-11-14T02:44:43,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/ec6c60e99bcd44b19308c4c632d24a5f to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/ec6c60e99bcd44b19308c4c632d24a5f 2024-11-14T02:44:43,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/8df6be96a30242eca9c2e6d10d273b1a to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/8df6be96a30242eca9c2e6d10d273b1a 2024-11-14T02:44:43,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/fd413c2308cc4dc6a21936275be9357e to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/fd413c2308cc4dc6a21936275be9357e 2024-11-14T02:44:43,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/06e8d02a1a164c55b871bd5a6023a738 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/06e8d02a1a164c55b871bd5a6023a738 2024-11-14T02:44:43,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/b8aa9ee8571646c494a1a75c2c774f0b to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/b8aa9ee8571646c494a1a75c2c774f0b 2024-11-14T02:44:43,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/c05d9389d8ca4f5e8ef85c43386f2c24 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/c05d9389d8ca4f5e8ef85c43386f2c24 2024-11-14T02:44:43,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/fa200a6b09c24fa5b65fcf9e3572d689 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/fa200a6b09c24fa5b65fcf9e3572d689 2024-11-14T02:44:43,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/fa8fc185752447c69e93517de923a7bf to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/fa8fc185752447c69e93517de923a7bf 2024-11-14T02:44:43,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/e4f842ffe48140319dff2003be6b3d51 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/e4f842ffe48140319dff2003be6b3d51 2024-11-14T02:44:43,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/5b0b8f86f44c4b18a17d0516cf50a046 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/5b0b8f86f44c4b18a17d0516cf50a046 2024-11-14T02:44:43,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/8e4972e7d972404da887ac26deee82e5 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/8e4972e7d972404da887ac26deee82e5 2024-11-14T02:44:43,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/7b66c03b38de426fae42abe6051b89cc to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/7b66c03b38de426fae42abe6051b89cc 2024-11-14T02:44:43,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/07ef0837cd2044558af5d7c8a6c78a07 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/07ef0837cd2044558af5d7c8a6c78a07 2024-11-14T02:44:43,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/b3151c2ce5754fff82366b4033ddf1bf to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/b3151c2ce5754fff82366b4033ddf1bf 2024-11-14T02:44:43,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/ac6c16ede1b34ae2a73922435451c173 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/ac6c16ede1b34ae2a73922435451c173 2024-11-14T02:44:43,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/e0306a004263431c95a794387e6185dc to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/e0306a004263431c95a794387e6185dc 2024-11-14T02:44:43,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/eea8ff59bad840298c506ab6f5fd7777, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/5e9ed33de6844cac8ffa291fe386d47e, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/c07541a9038641a08d4a87101fbd7b3a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/ea978eaced284ac4ac5df231ad0db4bc, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/cf9f2d4640d345c681804fd2693eebc0, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/28947feb082a46a5a5f492a98e7e33b9, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/b7d14de88129413a9f92641a18cc1923, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/87557d0c53fb4ddba1512d31c5cb8d33, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/ed909a2e648c46cd8cfaac3ee9dfd50b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/76096481c8df41cc89fa38f12b40762e, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/96810c57edbc4292a44a225b60c77886, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/e833ee1baf2f416688359f6df8321d24, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/af34d8a814a14e9cad8b96b313bdac8f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/6992d725bbe5430eb80afeeb7086b195, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/70142c65140449dca0d33d37e387dd4a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/d9855367b1734580b11ee9dc8d6bae1d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/fca40485f2e54918b972ba20ba31f2ab, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/3ae1e8c646d24600ae4f55ed94cef86e, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/4d4b5f2e03834c3d8ceb1d1959f13832, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/0cd9b65de0f842abbf87c8db6096e702, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/7e4a78bf09f04c7db8681ec5d8ca6d15] to archive 2024-11-14T02:44:43,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:44:43,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/eea8ff59bad840298c506ab6f5fd7777 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/eea8ff59bad840298c506ab6f5fd7777 2024-11-14T02:44:43,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/5e9ed33de6844cac8ffa291fe386d47e to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/5e9ed33de6844cac8ffa291fe386d47e 2024-11-14T02:44:43,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/c07541a9038641a08d4a87101fbd7b3a to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/c07541a9038641a08d4a87101fbd7b3a 2024-11-14T02:44:43,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/ea978eaced284ac4ac5df231ad0db4bc to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/ea978eaced284ac4ac5df231ad0db4bc 2024-11-14T02:44:43,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/cf9f2d4640d345c681804fd2693eebc0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/cf9f2d4640d345c681804fd2693eebc0 2024-11-14T02:44:43,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/28947feb082a46a5a5f492a98e7e33b9 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/28947feb082a46a5a5f492a98e7e33b9 2024-11-14T02:44:43,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/b7d14de88129413a9f92641a18cc1923 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/b7d14de88129413a9f92641a18cc1923 2024-11-14T02:44:43,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/87557d0c53fb4ddba1512d31c5cb8d33 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/87557d0c53fb4ddba1512d31c5cb8d33 2024-11-14T02:44:43,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/ed909a2e648c46cd8cfaac3ee9dfd50b to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/ed909a2e648c46cd8cfaac3ee9dfd50b 2024-11-14T02:44:43,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/76096481c8df41cc89fa38f12b40762e to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/76096481c8df41cc89fa38f12b40762e 2024-11-14T02:44:43,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/96810c57edbc4292a44a225b60c77886 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/96810c57edbc4292a44a225b60c77886 2024-11-14T02:44:43,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/e833ee1baf2f416688359f6df8321d24 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/e833ee1baf2f416688359f6df8321d24 2024-11-14T02:44:43,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/af34d8a814a14e9cad8b96b313bdac8f to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/af34d8a814a14e9cad8b96b313bdac8f 2024-11-14T02:44:43,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/6992d725bbe5430eb80afeeb7086b195 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/6992d725bbe5430eb80afeeb7086b195 2024-11-14T02:44:43,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/70142c65140449dca0d33d37e387dd4a to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/70142c65140449dca0d33d37e387dd4a 2024-11-14T02:44:43,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/d9855367b1734580b11ee9dc8d6bae1d to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/d9855367b1734580b11ee9dc8d6bae1d 2024-11-14T02:44:43,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/fca40485f2e54918b972ba20ba31f2ab to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/fca40485f2e54918b972ba20ba31f2ab 2024-11-14T02:44:43,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/3ae1e8c646d24600ae4f55ed94cef86e to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/3ae1e8c646d24600ae4f55ed94cef86e 2024-11-14T02:44:43,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/4d4b5f2e03834c3d8ceb1d1959f13832 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/4d4b5f2e03834c3d8ceb1d1959f13832 2024-11-14T02:44:43,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/0cd9b65de0f842abbf87c8db6096e702 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/0cd9b65de0f842abbf87c8db6096e702 2024-11-14T02:44:43,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/7e4a78bf09f04c7db8681ec5d8ca6d15 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/7e4a78bf09f04c7db8681ec5d8ca6d15 2024-11-14T02:44:43,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/1c9b5d759ca54d65b8acff3536ae47fb, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/d65dd6181b4f44528e77f3d92812fa3a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c8254b159900496aa57adf2dc90e6dcd, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/4ad332b4aa6b4863884af9cafebceaa0, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c61d03b8d91b4b16b12327ca3ef39763, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/1359f341464344de84da785b3f2c6747, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/79831d676e40413b9b1a4ded2438ad0f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/275f48f0405845738b627e7a2949a464, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/8866d91ff0c64dafbdb6f82a70a98a21, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c8d9b49a0afd4b0486c51ff9bb8b1cc9, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c668375d881a4bfbb5c7593b8eb575c2, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/00ffb42b7c7f4e74a3b2ab091a3a2995, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/84db99a6c13346e1b20bd96818f0eb43, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/246c2f31348849978670d51270cf11b1, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/84569c3101964c1b9696f2efdc35ed96, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/544060562cd943db92542c043deba15b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/847947259e154cf099d72adfec4c38fb, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/22295686b7724613949c6fdd4f710b6b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c59485d288844618b2af73e9723078c0, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/81da659d582a4cd9aa28d5e6e562894d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/2e14fc56237e422eb53adc561dff37f6] to archive 2024-11-14T02:44:43,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:44:43,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/1c9b5d759ca54d65b8acff3536ae47fb to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/1c9b5d759ca54d65b8acff3536ae47fb 2024-11-14T02:44:43,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/d65dd6181b4f44528e77f3d92812fa3a to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/d65dd6181b4f44528e77f3d92812fa3a 2024-11-14T02:44:43,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c8254b159900496aa57adf2dc90e6dcd to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c8254b159900496aa57adf2dc90e6dcd 2024-11-14T02:44:43,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/4ad332b4aa6b4863884af9cafebceaa0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/4ad332b4aa6b4863884af9cafebceaa0 2024-11-14T02:44:43,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c61d03b8d91b4b16b12327ca3ef39763 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c61d03b8d91b4b16b12327ca3ef39763 2024-11-14T02:44:43,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/1359f341464344de84da785b3f2c6747 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/1359f341464344de84da785b3f2c6747 2024-11-14T02:44:43,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/79831d676e40413b9b1a4ded2438ad0f to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/79831d676e40413b9b1a4ded2438ad0f 2024-11-14T02:44:43,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/275f48f0405845738b627e7a2949a464 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/275f48f0405845738b627e7a2949a464 2024-11-14T02:44:43,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/8866d91ff0c64dafbdb6f82a70a98a21 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/8866d91ff0c64dafbdb6f82a70a98a21 2024-11-14T02:44:43,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c8d9b49a0afd4b0486c51ff9bb8b1cc9 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c8d9b49a0afd4b0486c51ff9bb8b1cc9 2024-11-14T02:44:43,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c668375d881a4bfbb5c7593b8eb575c2 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c668375d881a4bfbb5c7593b8eb575c2 2024-11-14T02:44:43,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/00ffb42b7c7f4e74a3b2ab091a3a2995 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/00ffb42b7c7f4e74a3b2ab091a3a2995 2024-11-14T02:44:43,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/84db99a6c13346e1b20bd96818f0eb43 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/84db99a6c13346e1b20bd96818f0eb43 2024-11-14T02:44:43,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/246c2f31348849978670d51270cf11b1 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/246c2f31348849978670d51270cf11b1 2024-11-14T02:44:43,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/84569c3101964c1b9696f2efdc35ed96 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/84569c3101964c1b9696f2efdc35ed96 2024-11-14T02:44:43,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/544060562cd943db92542c043deba15b to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/544060562cd943db92542c043deba15b 2024-11-14T02:44:43,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/847947259e154cf099d72adfec4c38fb to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/847947259e154cf099d72adfec4c38fb 2024-11-14T02:44:43,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/22295686b7724613949c6fdd4f710b6b to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/22295686b7724613949c6fdd4f710b6b 2024-11-14T02:44:43,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c59485d288844618b2af73e9723078c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/c59485d288844618b2af73e9723078c0 2024-11-14T02:44:43,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/81da659d582a4cd9aa28d5e6e562894d to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/81da659d582a4cd9aa28d5e6e562894d 2024-11-14T02:44:43,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/2e14fc56237e422eb53adc561dff37f6 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/2e14fc56237e422eb53adc561dff37f6 2024-11-14T02:44:43,683 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/recovered.edits/334.seqid, newMaxSeqId=334, maxSeqId=1 2024-11-14T02:44:43,684 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. 2024-11-14T02:44:43,684 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for 35a73edee503a7eb9039857e3cfc97f1: Waiting for close lock at 1731552282299Running coprocessor pre-close hooks at 1731552282299Disabling compacts and flushes for region at 1731552282299Disabling writes for close at 1731552282299Obtaining lock to block concurrent updates at 1731552282300 (+1 ms)Preparing flush snapshotting stores in 35a73edee503a7eb9039857e3cfc97f1 at 1731552282300Finished memstore snapshotting TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731552282301 (+1 ms)Flushing stores of TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1. at 1731552282302 (+1 ms)Flushing 35a73edee503a7eb9039857e3cfc97f1/A: creating writer at 1731552282302Flushing 35a73edee503a7eb9039857e3cfc97f1/A: appending metadata at 1731552282309 (+7 ms)Flushing 35a73edee503a7eb9039857e3cfc97f1/A: closing flushed file at 1731552282309Flushing 35a73edee503a7eb9039857e3cfc97f1/B: creating writer at 1731552282725 (+416 ms)Flushing 35a73edee503a7eb9039857e3cfc97f1/B: appending metadata at 1731552282728 (+3 ms)Flushing 35a73edee503a7eb9039857e3cfc97f1/B: closing flushed file at 1731552282728Flushing 35a73edee503a7eb9039857e3cfc97f1/C: creating writer at 1731552283137 (+409 ms)Flushing 35a73edee503a7eb9039857e3cfc97f1/C: appending metadata at 1731552283139 (+2 ms)Flushing 35a73edee503a7eb9039857e3cfc97f1/C: closing flushed file at 1731552283139Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2cdfe256: reopening flushed file at 1731552283584 (+445 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31f9b8ee: reopening flushed file at 1731552283593 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41a2135c: reopening flushed file at 1731552283599 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 35a73edee503a7eb9039857e3cfc97f1 in 1307ms, sequenceid=331, compaction requested=true at 1731552283606 (+7 ms)Writing region close event to WAL at 1731552283681 (+75 ms)Running coprocessor post-close hooks at 1731552283684 (+3 ms)Closed at 1731552283684 2024-11-14T02:44:43,686 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed 35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:43,686 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=35a73edee503a7eb9039857e3cfc97f1, regionState=CLOSED 2024-11-14T02:44:43,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 because future has completed 2024-11-14T02:44:43,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-14T02:44:43,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure 35a73edee503a7eb9039857e3cfc97f1, server=83c5a5c119d5,41455,1731552191259 in 1.5430 sec 2024-11-14T02:44:43,690 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-14T02:44:43,690 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=35a73edee503a7eb9039857e3cfc97f1, UNASSIGN in 1.5480 sec 2024-11-14T02:44:43,692 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-14T02:44:43,692 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5520 sec 2024-11-14T02:44:43,693 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552283693"}]},"ts":"1731552283693"} 2024-11-14T02:44:43,694 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T02:44:43,694 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T02:44:43,695 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5630 sec 2024-11-14T02:44:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-14T02:44:44,285 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:44,286 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T02:44:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:44,291 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-14T02:44:44,293 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=88, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:44,297 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:44,302 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A, FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B, FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C, FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/recovered.edits] 2024-11-14T02:44:44,305 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/2a8e93fbfc384a29be5c733be3661bca to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/2a8e93fbfc384a29be5c733be3661bca 2024-11-14T02:44:44,306 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/83aa231be600441885674e1ff254cbc5 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/83aa231be600441885674e1ff254cbc5 2024-11-14T02:44:44,307 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/c903eccf647f4a478e3a87ba0337e4c2 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/A/c903eccf647f4a478e3a87ba0337e4c2 2024-11-14T02:44:44,308 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/8d001d7d7af54e77aeb8066826af85d5 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/8d001d7d7af54e77aeb8066826af85d5 2024-11-14T02:44:44,309 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/c461d68aec9844f4bc489ff0960aae10 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/c461d68aec9844f4bc489ff0960aae10 2024-11-14T02:44:44,310 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/f1885a39dca04c4190283846e667fc51 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/B/f1885a39dca04c4190283846e667fc51 2024-11-14T02:44:44,312 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/2c0efec743ab4491a3016740c4d4bc0a to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/2c0efec743ab4491a3016740c4d4bc0a 2024-11-14T02:44:44,313 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/67e8d3f2920d41119f52dace84ccdabf to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/67e8d3f2920d41119f52dace84ccdabf 2024-11-14T02:44:44,313 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/bf7454df70e24d76acf8335b56306708 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/C/bf7454df70e24d76acf8335b56306708 2024-11-14T02:44:44,316 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/recovered.edits/334.seqid to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1/recovered.edits/334.seqid 2024-11-14T02:44:44,316 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/35a73edee503a7eb9039857e3cfc97f1 2024-11-14T02:44:44,316 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T02:44:44,318 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=88, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:44,321 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T02:44:44,323 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T02:44:44,324 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=88, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:44,324 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T02:44:44,324 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731552284324"}]},"ts":"9223372036854775807"} 2024-11-14T02:44:44,326 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T02:44:44,326 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 35a73edee503a7eb9039857e3cfc97f1, NAME => 'TestAcidGuarantees,,1731552251544.35a73edee503a7eb9039857e3cfc97f1.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T02:44:44,326 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T02:44:44,326 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731552284326"}]},"ts":"9223372036854775807"} 2024-11-14T02:44:44,328 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T02:44:44,328 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=88, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:44,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 42 msec 2024-11-14T02:44:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-14T02:44:44,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T02:44:44,404 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:44,411 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238 (was 242), OpenFileDescriptor=447 (was 463), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=297 (was 346), ProcessCount=11 (was 11), AvailableMemoryMB=5321 (was 5374) 2024-11-14T02:44:44,419 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=238, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=297, ProcessCount=11, AvailableMemoryMB=5321 2024-11-14T02:44:44,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T02:44:44,420 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T02:44:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:44,422 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T02:44:44,422 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:44,422 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 89 2024-11-14T02:44:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T02:44:44,423 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T02:44:44,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742127_1303 (size=959) 2024-11-14T02:44:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T02:44:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T02:44:44,835 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b 2024-11-14T02:44:44,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742128_1304 (size=53) 2024-11-14T02:44:45,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T02:44:45,248 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:44:45,248 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 025639fc1f987c76e816bd52e91790c0, disabling compactions & flushes 2024-11-14T02:44:45,248 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:45,248 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:45,248 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. after waiting 0 ms 2024-11-14T02:44:45,248 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:45,248 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:45,249 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 025639fc1f987c76e816bd52e91790c0: Waiting for close lock at 1731552285248Disabling compacts and flushes for region at 1731552285248Disabling writes for close at 1731552285248Writing region close event to WAL at 1731552285248Closed at 1731552285248 2024-11-14T02:44:45,252 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T02:44:45,253 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731552285252"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731552285252"}]},"ts":"1731552285252"} 2024-11-14T02:44:45,257 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T02:44:45,258 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T02:44:45,258 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552285258"}]},"ts":"1731552285258"} 2024-11-14T02:44:45,260 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T02:44:45,260 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=025639fc1f987c76e816bd52e91790c0, ASSIGN}] 2024-11-14T02:44:45,262 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=025639fc1f987c76e816bd52e91790c0, ASSIGN 2024-11-14T02:44:45,263 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=025639fc1f987c76e816bd52e91790c0, ASSIGN; state=OFFLINE, location=83c5a5c119d5,41455,1731552191259; forceNewPlan=false, retain=false 2024-11-14T02:44:45,414 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=025639fc1f987c76e816bd52e91790c0, regionState=OPENING, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:45,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=025639fc1f987c76e816bd52e91790c0, ASSIGN because future has completed 2024-11-14T02:44:45,419 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259}] 2024-11-14T02:44:45,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T02:44:45,580 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:45,581 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7752): Opening region: {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:44:45,581 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:45,581 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:44:45,581 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7794): checking encryption for 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:45,581 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7797): checking classloading for 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:45,583 INFO [StoreOpener-025639fc1f987c76e816bd52e91790c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:45,585 INFO [StoreOpener-025639fc1f987c76e816bd52e91790c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 025639fc1f987c76e816bd52e91790c0 columnFamilyName A 2024-11-14T02:44:45,586 DEBUG [StoreOpener-025639fc1f987c76e816bd52e91790c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:45,586 INFO [StoreOpener-025639fc1f987c76e816bd52e91790c0-1 {}] regionserver.HStore(327): Store=025639fc1f987c76e816bd52e91790c0/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:44:45,587 INFO [StoreOpener-025639fc1f987c76e816bd52e91790c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:45,588 INFO [StoreOpener-025639fc1f987c76e816bd52e91790c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 025639fc1f987c76e816bd52e91790c0 columnFamilyName B 2024-11-14T02:44:45,588 DEBUG [StoreOpener-025639fc1f987c76e816bd52e91790c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:45,589 INFO [StoreOpener-025639fc1f987c76e816bd52e91790c0-1 {}] regionserver.HStore(327): Store=025639fc1f987c76e816bd52e91790c0/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:44:45,589 INFO [StoreOpener-025639fc1f987c76e816bd52e91790c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:45,591 INFO [StoreOpener-025639fc1f987c76e816bd52e91790c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 025639fc1f987c76e816bd52e91790c0 columnFamilyName C 2024-11-14T02:44:45,591 DEBUG [StoreOpener-025639fc1f987c76e816bd52e91790c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:45,592 INFO [StoreOpener-025639fc1f987c76e816bd52e91790c0-1 {}] regionserver.HStore(327): Store=025639fc1f987c76e816bd52e91790c0/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:44:45,592 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1038): replaying wal for 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:45,593 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:45,593 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:45,594 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1048): stopping wal replay for 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:45,595 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1060): Cleaning up temporary data for 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:45,595 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T02:44:45,597 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1093): writing seq id for 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:45,599 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T02:44:45,600 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1114): Opened 025639fc1f987c76e816bd52e91790c0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64882875, jitterRate=-0.03316982090473175}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T02:44:45,600 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:45,601 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1006): Region open journal for 025639fc1f987c76e816bd52e91790c0: Running coprocessor pre-open hook at 1731552285582Writing region info on filesystem at 1731552285582Initializing all the Stores at 1731552285583 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552285583Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552285583Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552285583Cleaning up temporary data from old regions at 1731552285595 (+12 ms)Running coprocessor post-open hooks at 1731552285600 (+5 ms)Region opened successfully at 1731552285601 (+1 ms) 2024-11-14T02:44:45,602 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., pid=91, masterSystemTime=1731552285573 2024-11-14T02:44:45,603 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:45,603 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:45,604 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=025639fc1f987c76e816bd52e91790c0, regionState=OPEN, openSeqNum=2, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:45,605 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 because future has completed 2024-11-14T02:44:45,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-14T02:44:45,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 in 187 msec 2024-11-14T02:44:45,610 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-14T02:44:45,610 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=025639fc1f987c76e816bd52e91790c0, ASSIGN in 348 msec 2024-11-14T02:44:45,611 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T02:44:45,611 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552285611"}]},"ts":"1731552285611"} 2024-11-14T02:44:45,613 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T02:44:45,613 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T02:44:45,615 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1930 sec 2024-11-14T02:44:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T02:44:46,585 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:46,588 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283b5885, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:46,588 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:44:46,588 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:46,590 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:44:46,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:46,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:44:46,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23b5a2cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:46,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:44:46,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:46,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:46,594 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:46,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e5a24b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:46,596 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:46,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:44:46,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:46,598 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:46,599 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T02:44:46,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 83c5a5c119d5,43239,1731552190584 2024-11-14T02:44:46,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@75c246de 2024-11-14T02:44:46,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T02:44:46,601 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T02:44:46,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T02:44:46,604 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T02:44:46,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:46,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742129_1305 (size=995) 2024-11-14T02:44:47,021 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-14T02:44:47,021 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-14T02:44:47,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T02:44:47,030 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=025639fc1f987c76e816bd52e91790c0, REOPEN/MOVE}] 2024-11-14T02:44:47,031 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=025639fc1f987c76e816bd52e91790c0, REOPEN/MOVE 2024-11-14T02:44:47,032 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=025639fc1f987c76e816bd52e91790c0, regionState=CLOSING, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:47,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=025639fc1f987c76e816bd52e91790c0, REOPEN/MOVE because future has completed 2024-11-14T02:44:47,034 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T02:44:47,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259}] 2024-11-14T02:44:47,186 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:47,186 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T02:44:47,186 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing 025639fc1f987c76e816bd52e91790c0, disabling compactions & flushes 2024-11-14T02:44:47,186 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:47,186 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:47,186 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. after waiting 0 ms 2024-11-14T02:44:47,186 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:47,194 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T02:44:47,195 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:47,195 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for 025639fc1f987c76e816bd52e91790c0: Waiting for close lock at 1731552287186Running coprocessor pre-close hooks at 1731552287186Disabling compacts and flushes for region at 1731552287186Disabling writes for close at 1731552287186Writing region close event to WAL at 1731552287187 (+1 ms)Running coprocessor post-close hooks at 1731552287195 (+8 ms)Closed at 1731552287195 2024-11-14T02:44:47,196 WARN [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionServer(3299): Not adding moved region record: 025639fc1f987c76e816bd52e91790c0 to self. 2024-11-14T02:44:47,200 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:47,201 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=025639fc1f987c76e816bd52e91790c0, regionState=CLOSED 2024-11-14T02:44:47,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 because future has completed 2024-11-14T02:44:47,208 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-14T02:44:47,208 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure 025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 in 170 msec 2024-11-14T02:44:47,209 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=025639fc1f987c76e816bd52e91790c0, REOPEN/MOVE; state=CLOSED, location=83c5a5c119d5,41455,1731552191259; forceNewPlan=false, retain=true 2024-11-14T02:44:47,360 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=025639fc1f987c76e816bd52e91790c0, regionState=OPENING, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:47,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=025639fc1f987c76e816bd52e91790c0, REOPEN/MOVE because future has completed 2024-11-14T02:44:47,363 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259}] 2024-11-14T02:44:47,524 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:47,525 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7752): Opening region: {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:44:47,525 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:47,526 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:44:47,526 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7794): checking encryption for 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:47,526 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7797): checking classloading for 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:47,529 INFO [StoreOpener-025639fc1f987c76e816bd52e91790c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:47,529 INFO [StoreOpener-025639fc1f987c76e816bd52e91790c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 025639fc1f987c76e816bd52e91790c0 columnFamilyName A 2024-11-14T02:44:47,531 DEBUG [StoreOpener-025639fc1f987c76e816bd52e91790c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:47,531 INFO [StoreOpener-025639fc1f987c76e816bd52e91790c0-1 {}] regionserver.HStore(327): Store=025639fc1f987c76e816bd52e91790c0/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:44:47,532 INFO [StoreOpener-025639fc1f987c76e816bd52e91790c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:47,532 INFO [StoreOpener-025639fc1f987c76e816bd52e91790c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 025639fc1f987c76e816bd52e91790c0 columnFamilyName B 2024-11-14T02:44:47,532 DEBUG [StoreOpener-025639fc1f987c76e816bd52e91790c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:47,533 INFO [StoreOpener-025639fc1f987c76e816bd52e91790c0-1 {}] regionserver.HStore(327): Store=025639fc1f987c76e816bd52e91790c0/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:44:47,533 INFO [StoreOpener-025639fc1f987c76e816bd52e91790c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:47,534 INFO [StoreOpener-025639fc1f987c76e816bd52e91790c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 025639fc1f987c76e816bd52e91790c0 columnFamilyName C 2024-11-14T02:44:47,534 DEBUG [StoreOpener-025639fc1f987c76e816bd52e91790c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:47,534 INFO [StoreOpener-025639fc1f987c76e816bd52e91790c0-1 {}] regionserver.HStore(327): Store=025639fc1f987c76e816bd52e91790c0/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:44:47,534 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1038): replaying wal for 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:47,535 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:47,537 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:47,538 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1048): stopping wal replay for 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:47,538 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1060): Cleaning up temporary data for 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:47,538 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T02:44:47,540 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1093): writing seq id for 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:47,541 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1114): Opened 025639fc1f987c76e816bd52e91790c0; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65724893, jitterRate=-0.020622774958610535}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T02:44:47,541 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:47,542 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1006): Region open journal for 025639fc1f987c76e816bd52e91790c0: Running coprocessor pre-open hook at 1731552287526Writing region info on filesystem at 1731552287527 (+1 ms)Initializing all the Stores at 1731552287528 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552287528Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552287528Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552287528Cleaning up temporary data from old regions at 1731552287538 (+10 ms)Running coprocessor post-open hooks at 1731552287541 (+3 ms)Region opened successfully at 1731552287542 (+1 ms) 2024-11-14T02:44:47,543 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., pid=96, masterSystemTime=1731552287517 2024-11-14T02:44:47,545 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:47,545 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:47,546 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=025639fc1f987c76e816bd52e91790c0, regionState=OPEN, openSeqNum=5, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:47,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 because future has completed 2024-11-14T02:44:47,552 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=94 2024-11-14T02:44:47,552 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 in 186 msec 2024-11-14T02:44:47,554 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-14T02:44:47,554 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=025639fc1f987c76e816bd52e91790c0, REOPEN/MOVE in 522 msec 2024-11-14T02:44:47,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-14T02:44:47,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 527 msec 2024-11-14T02:44:47,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 953 msec 2024-11-14T02:44:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-14T02:44:47,562 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:47,563 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,563 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:44:47,563 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:47,565 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:44:47,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:47,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:44:47,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3335bcf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:44:47,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:47,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:47,567 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:47,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33588b9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,568 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:44:47,569 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:47,570 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:44:47,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:47,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:44:47,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bf7f5d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:44:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:47,572 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:47,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77cf2d0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,573 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:44:47,574 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:47,574 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:44:47,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:47,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:44:47,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@351cb63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:44:47,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:47,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:47,576 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:47,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d1a2287, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,578 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5345722e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,578 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:44:47,578 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:47,579 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:44:47,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:47,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:44:47,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54dc5949, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:44:47,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:47,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:47,581 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:47,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3843c754, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd3a90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,583 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:44:47,583 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:47,584 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:44:47,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:47,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:44:47,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ab2576, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:44:47,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:47,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:47,586 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:47,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b51378, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,587 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eba3e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,587 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:44:47,587 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:47,588 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:44:47,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:47,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:44:47,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e15f3e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:44:47,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:47,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:47,590 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:47,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ef504c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,591 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79570776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,591 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:44:47,592 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:47,592 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:44:47,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:47,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:44:47,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@482db613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:44:47,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:47,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:47,594 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:47,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e780c0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,595 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43065a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,595 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:44:47,595 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:47,596 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:44:47,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:47,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:44:47,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8d955f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:44:47,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:47,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:47,597 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:47,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b458b6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,599 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bcd92c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,599 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:44:47,599 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:47,599 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:44:47,600 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:47,600 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:44:47,600 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6047d1f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,600 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:44:47,600 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:47,600 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:47,601 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:47,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78612772, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,602 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b779047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,602 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:44:47,602 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:47,603 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:44:47,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:47,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:44:47,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f19ec41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:44:47,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:47,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:47,605 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:47,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61698f19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:47,606 DEBUG [Thread-1415 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:47,606 DEBUG [Thread-1426 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:47,606 DEBUG [Thread-1428 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:47,606 DEBUG [Thread-1417 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:47,606 DEBUG [Thread-1420 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:47,607 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:47,607 DEBUG [Thread-1422 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:47,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:44:47,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:44:47,607 DEBUG [Thread-1415 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:44:47,607 DEBUG [Thread-1428 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:44:47,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:47,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:47,607 DEBUG [Thread-1415 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:47,607 DEBUG [Thread-1428 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:47,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:44:47,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:47,608 DEBUG [Thread-1409 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:47,608 DEBUG [Thread-1411 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:47,608 DEBUG [Thread-1413 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:47,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:44:47,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:44:47,609 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:47,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:47,609 DEBUG [Thread-1420 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:44:47,609 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:47,609 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:47,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:47,609 DEBUG [Thread-1420 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:47,609 DEBUG [Thread-1409 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:44:47,609 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:47,609 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:47,609 DEBUG [Thread-1409 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:47,610 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:47,610 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:47,610 DEBUG [Thread-1424 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:47,610 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-14T02:44:47,610 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:47,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:44:47,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:44:47,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:47,611 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:47,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:44:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T02:44:47,613 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:47,613 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:47,613 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:44:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:44:47,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:44:47,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:44:47,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:44:47,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:44:47,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:44:47,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:44:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:47,624 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:44:47,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 025639fc1f987c76e816bd52e91790c0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T02:44:47,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111403204f564ab54e59b8669350f37e7826_025639fc1f987c76e816bd52e91790c0 is 50, key is test_row_0/A:col10/1731552287624/Put/seqid=0 2024-11-14T02:44:47,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742130_1306 (size=14594) 2024-11-14T02:44:47,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552347661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:47,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:47,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552347664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:47,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:47,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:47,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552347670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:47,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552347671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:47,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:47,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552347671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:47,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T02:44:47,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T02:44:47,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:47,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:47,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:47,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:47,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:47,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552347783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:47,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:47,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552347784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:47,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:47,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552347784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:47,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:47,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552347784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:47,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:47,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552347784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:47,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T02:44:47,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:47,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:47,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:47,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:47,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:47,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T02:44:47,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:47,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552347994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:47,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:47,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:47,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552347994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:47,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:47,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:47,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552347994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:47,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:47,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552347994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:47,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:47,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:47,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552347994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:47,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:48,058 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:48,061 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111403204f564ab54e59b8669350f37e7826_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111403204f564ab54e59b8669350f37e7826_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:48,061 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/d8f61c14e4ed4b2c8aa30e46b2cbb763, store: [table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:44:48,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/d8f61c14e4ed4b2c8aa30e46b2cbb763 is 175, key is test_row_0/A:col10/1731552287624/Put/seqid=0 2024-11-14T02:44:48,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742131_1307 (size=39549) 2024-11-14T02:44:48,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T02:44:48,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:48,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:48,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:48,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:48,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:48,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:48,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T02:44:48,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:48,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:48,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:48,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:48,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:48,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T02:44:48,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:48,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552348303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:48,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:48,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:48,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:48,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552348304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:48,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:48,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:48,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:48,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:48,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552348304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:48,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:48,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:48,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:48,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552348304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:48,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:48,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:48,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:48,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:48,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552348304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:48,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:48,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:48,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:48,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T02:44:48,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:48,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:48,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:48,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:48,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:48,466 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/d8f61c14e4ed4b2c8aa30e46b2cbb763 2024-11-14T02:44:48,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/10f55b3e62764b8cafda603ccc41e447 is 50, key is test_row_0/B:col10/1731552287624/Put/seqid=0 2024-11-14T02:44:48,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742132_1308 (size=12001) 2024-11-14T02:44:48,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T02:44:48,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:48,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:48,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:48,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:48,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:48,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T02:44:48,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:48,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:48,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:48,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:48,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:48,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:48,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T02:44:48,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:48,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552348813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:48,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:48,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:48,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:48,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:48,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552348813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:48,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:48,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:48,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:48,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:48,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552348813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:48,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:48,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:48,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:48,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552348813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:48,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:48,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:48,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:48,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:48,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552348823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:48,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:48,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:48,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:48,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T02:44:48,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:48,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:48,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:48,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:48,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:48,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/10f55b3e62764b8cafda603ccc41e447 2024-11-14T02:44:48,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/4c2837c1705248f2b185603d75e6dafd is 50, key is test_row_0/C:col10/1731552287624/Put/seqid=0 2024-11-14T02:44:48,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742133_1309 (size=12001) 2024-11-14T02:44:48,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/4c2837c1705248f2b185603d75e6dafd 2024-11-14T02:44:48,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/d8f61c14e4ed4b2c8aa30e46b2cbb763 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d8f61c14e4ed4b2c8aa30e46b2cbb763 2024-11-14T02:44:48,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d8f61c14e4ed4b2c8aa30e46b2cbb763, entries=200, sequenceid=17, filesize=38.6 K 2024-11-14T02:44:48,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/10f55b3e62764b8cafda603ccc41e447 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/10f55b3e62764b8cafda603ccc41e447 2024-11-14T02:44:48,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/10f55b3e62764b8cafda603ccc41e447, entries=150, sequenceid=17, filesize=11.7 K 2024-11-14T02:44:48,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/4c2837c1705248f2b185603d75e6dafd as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/4c2837c1705248f2b185603d75e6dafd 2024-11-14T02:44:48,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/4c2837c1705248f2b185603d75e6dafd, entries=150, sequenceid=17, filesize=11.7 K 2024-11-14T02:44:48,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 025639fc1f987c76e816bd52e91790c0 in 1317ms, sequenceid=17, compaction requested=false 2024-11-14T02:44:48,941 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T02:44:48,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:44:48,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T02:44:48,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:48,990 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 025639fc1f987c76e816bd52e91790c0 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-14T02:44:48,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142e7c88664b6d44ef848d345305dbcddb_025639fc1f987c76e816bd52e91790c0 is 50, key is test_row_0/A:col10/1731552287660/Put/seqid=0 2024-11-14T02:44:49,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742134_1310 (size=12154) 2024-11-14T02:44:49,212 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T02:44:49,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:49,408 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142e7c88664b6d44ef848d345305dbcddb_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142e7c88664b6d44ef848d345305dbcddb_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:49,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/2fae77053b9847478b445a21a4dcb9a3, store: [table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:44:49,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/2fae77053b9847478b445a21a4dcb9a3 is 175, key is test_row_0/A:col10/1731552287660/Put/seqid=0 2024-11-14T02:44:49,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742135_1311 (size=30955) 2024-11-14T02:44:49,418 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/2fae77053b9847478b445a21a4dcb9a3 2024-11-14T02:44:49,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/0a6e2743d51849e4b1cb457a8fc8e613 is 50, key is test_row_0/B:col10/1731552287660/Put/seqid=0 2024-11-14T02:44:49,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742136_1312 (size=12001) 2024-11-14T02:44:49,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T02:44:49,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:49,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:49,828 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/0a6e2743d51849e4b1cb457a8fc8e613 2024-11-14T02:44:49,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/dd69135deab94cb996c1ee399f3af4c3 is 50, key is test_row_0/C:col10/1731552287660/Put/seqid=0 2024-11-14T02:44:49,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742137_1313 (size=12001) 2024-11-14T02:44:49,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:49,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552349836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:49,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:49,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:49,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552349837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:49,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:49,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552349838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:49,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:49,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:49,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552349838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:49,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:49,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552349840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:49,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:49,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:49,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552349943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:49,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:49,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552349953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552349953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552349953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:49,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:49,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:49,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552349953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:49,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:50,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:50,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552350153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:50,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:50,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552350163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:50,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:50,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:50,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552350163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:50,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:50,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552350163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:50,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:50,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552350164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:50,240 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/dd69135deab94cb996c1ee399f3af4c3 2024-11-14T02:44:50,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/2fae77053b9847478b445a21a4dcb9a3 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/2fae77053b9847478b445a21a4dcb9a3 2024-11-14T02:44:50,246 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/2fae77053b9847478b445a21a4dcb9a3, entries=150, sequenceid=41, filesize=30.2 K 2024-11-14T02:44:50,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/0a6e2743d51849e4b1cb457a8fc8e613 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/0a6e2743d51849e4b1cb457a8fc8e613 2024-11-14T02:44:50,250 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/0a6e2743d51849e4b1cb457a8fc8e613, entries=150, sequenceid=41, filesize=11.7 K 2024-11-14T02:44:50,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/dd69135deab94cb996c1ee399f3af4c3 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/dd69135deab94cb996c1ee399f3af4c3 2024-11-14T02:44:50,255 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/dd69135deab94cb996c1ee399f3af4c3, entries=150, sequenceid=41, filesize=11.7 K 2024-11-14T02:44:50,256 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 025639fc1f987c76e816bd52e91790c0 in 1266ms, sequenceid=41, compaction requested=false 2024-11-14T02:44:50,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:44:50,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:50,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-14T02:44:50,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-14T02:44:50,258 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-14T02:44:50,258 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6440 sec 2024-11-14T02:44:50,260 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.6520 sec 2024-11-14T02:44:50,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:50,465 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:44:50,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 025639fc1f987c76e816bd52e91790c0 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-14T02:44:50,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111469dd57fd436d48ec99e74bacee156616_025639fc1f987c76e816bd52e91790c0 is 50, key is test_row_0/A:col10/1731552289836/Put/seqid=0 2024-11-14T02:44:50,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742138_1314 (size=14594) 2024-11-14T02:44:50,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:50,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552350501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:50,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:50,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:50,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:50,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552350521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:50,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552350521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:50,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:50,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:50,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:50,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552350522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:50,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:50,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552350522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:50,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:50,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:50,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:50,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552350634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:50,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:50,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:50,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552350643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:50,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:50,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:50,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552350643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:50,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:50,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:50,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552350643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:50,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552350643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:50,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552350843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:50,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:50,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552350853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:50,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552350853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:50,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552350853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:50,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:50,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552350863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:50,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:50,876 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:50,880 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111469dd57fd436d48ec99e74bacee156616_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111469dd57fd436d48ec99e74bacee156616_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:50,880 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/7c6554028edf49369a25150271d2cba9, store: [table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:44:50,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/7c6554028edf49369a25150271d2cba9 is 175, key is test_row_0/A:col10/1731552289836/Put/seqid=0 2024-11-14T02:44:50,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742139_1315 (size=39549) 2024-11-14T02:44:51,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552351154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:51,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:51,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552351163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:51,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:51,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552351163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:51,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:51,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552351163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:51,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:51,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552351173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:51,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:51,284 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/7c6554028edf49369a25150271d2cba9 2024-11-14T02:44:51,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/e0841ab766254b9186b776846b87d760 is 50, key is test_row_0/B:col10/1731552289836/Put/seqid=0 2024-11-14T02:44:51,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742140_1316 (size=12001) 2024-11-14T02:44:51,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552351673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:51,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:51,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552351673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:51,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:51,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:51,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552351673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:51,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:51,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:51,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552351673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:51,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:51,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552351683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:51,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/e0841ab766254b9186b776846b87d760 2024-11-14T02:44:51,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/155b8be1dc3244eab456afd8e5d61c91 is 50, key is test_row_0/C:col10/1731552289836/Put/seqid=0 2024-11-14T02:44:51,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742141_1317 (size=12001) 2024-11-14T02:44:51,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/155b8be1dc3244eab456afd8e5d61c91 2024-11-14T02:44:51,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/7c6554028edf49369a25150271d2cba9 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/7c6554028edf49369a25150271d2cba9 2024-11-14T02:44:51,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/7c6554028edf49369a25150271d2cba9, entries=200, sequenceid=54, filesize=38.6 K 2024-11-14T02:44:51,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/e0841ab766254b9186b776846b87d760 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/e0841ab766254b9186b776846b87d760 2024-11-14T02:44:51,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/e0841ab766254b9186b776846b87d760, entries=150, sequenceid=54, filesize=11.7 K 2024-11-14T02:44:51,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/155b8be1dc3244eab456afd8e5d61c91 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/155b8be1dc3244eab456afd8e5d61c91 2024-11-14T02:44:51,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/155b8be1dc3244eab456afd8e5d61c91, entries=150, sequenceid=54, filesize=11.7 K 2024-11-14T02:44:51,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 025639fc1f987c76e816bd52e91790c0 in 1254ms, sequenceid=54, compaction requested=true 2024-11-14T02:44:51,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:44:51,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 025639fc1f987c76e816bd52e91790c0:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:44:51,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:51,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 025639fc1f987c76e816bd52e91790c0:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:44:51,720 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:51,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:51,720 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:51,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 025639fc1f987c76e816bd52e91790c0:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:44:51,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:51,720 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:51,720 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:51,721 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 025639fc1f987c76e816bd52e91790c0/A is initiating minor compaction (all files) 2024-11-14T02:44:51,721 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 025639fc1f987c76e816bd52e91790c0/B is initiating minor compaction (all files) 2024-11-14T02:44:51,721 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 025639fc1f987c76e816bd52e91790c0/B in TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:51,721 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 025639fc1f987c76e816bd52e91790c0/A in TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:51,721 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/10f55b3e62764b8cafda603ccc41e447, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/0a6e2743d51849e4b1cb457a8fc8e613, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/e0841ab766254b9186b776846b87d760] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp, totalSize=35.2 K 2024-11-14T02:44:51,721 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d8f61c14e4ed4b2c8aa30e46b2cbb763, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/2fae77053b9847478b445a21a4dcb9a3, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/7c6554028edf49369a25150271d2cba9] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp, totalSize=107.5 K 2024-11-14T02:44:51,721 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:51,721 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. files: [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d8f61c14e4ed4b2c8aa30e46b2cbb763, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/2fae77053b9847478b445a21a4dcb9a3, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/7c6554028edf49369a25150271d2cba9] 2024-11-14T02:44:51,721 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 10f55b3e62764b8cafda603ccc41e447, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731552287616 2024-11-14T02:44:51,721 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8f61c14e4ed4b2c8aa30e46b2cbb763, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731552287616 2024-11-14T02:44:51,721 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a6e2743d51849e4b1cb457a8fc8e613, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731552287660 2024-11-14T02:44:51,722 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fae77053b9847478b445a21a4dcb9a3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731552287660 2024-11-14T02:44:51,722 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting e0841ab766254b9186b776846b87d760, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731552289836 2024-11-14T02:44:51,722 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c6554028edf49369a25150271d2cba9, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731552289835 2024-11-14T02:44:51,732 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 025639fc1f987c76e816bd52e91790c0#B#compaction#269 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:51,732 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:44:51,733 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/e6064d0e2d394cf08c883deeb49b5317 is 50, key is test_row_0/B:col10/1731552289836/Put/seqid=0 2024-11-14T02:44:51,735 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114eb481380a86441da88fec6ed1e62ef23_025639fc1f987c76e816bd52e91790c0 store=[table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:44:51,737 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114eb481380a86441da88fec6ed1e62ef23_025639fc1f987c76e816bd52e91790c0, store=[table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:44:51,737 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114eb481380a86441da88fec6ed1e62ef23_025639fc1f987c76e816bd52e91790c0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:44:51,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742142_1318 (size=12104) 2024-11-14T02:44:51,746 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/e6064d0e2d394cf08c883deeb49b5317 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/e6064d0e2d394cf08c883deeb49b5317 2024-11-14T02:44:51,749 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 025639fc1f987c76e816bd52e91790c0/B of 025639fc1f987c76e816bd52e91790c0 into e6064d0e2d394cf08c883deeb49b5317(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:51,749 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:44:51,749 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., storeName=025639fc1f987c76e816bd52e91790c0/B, priority=13, startTime=1731552291720; duration=0sec 2024-11-14T02:44:51,749 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:51,749 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 025639fc1f987c76e816bd52e91790c0:B 2024-11-14T02:44:51,750 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:51,752 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:51,752 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 025639fc1f987c76e816bd52e91790c0/C is initiating minor compaction (all files) 2024-11-14T02:44:51,752 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 025639fc1f987c76e816bd52e91790c0/C in TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:51,753 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/4c2837c1705248f2b185603d75e6dafd, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/dd69135deab94cb996c1ee399f3af4c3, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/155b8be1dc3244eab456afd8e5d61c91] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp, totalSize=35.2 K 2024-11-14T02:44:51,753 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c2837c1705248f2b185603d75e6dafd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731552287616 2024-11-14T02:44:51,753 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting dd69135deab94cb996c1ee399f3af4c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731552287660 2024-11-14T02:44:51,754 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 155b8be1dc3244eab456afd8e5d61c91, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731552289836 2024-11-14T02:44:51,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742143_1319 (size=4469) 2024-11-14T02:44:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T02:44:51,774 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 025639fc1f987c76e816bd52e91790c0#A#compaction#270 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:51,774 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:51,774 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/2a064daf117a49e284fd0107acf6770c is 175, key is test_row_0/A:col10/1731552289836/Put/seqid=0 2024-11-14T02:44:51,774 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-14T02:44:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T02:44:51,777 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:51,777 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 025639fc1f987c76e816bd52e91790c0#C#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:51,777 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:51,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:51,778 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/8a6b3e767f2d40dc88a856971c266f27 is 50, key is test_row_0/C:col10/1731552289836/Put/seqid=0 2024-11-14T02:44:51,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742144_1320 (size=31058) 2024-11-14T02:44:51,816 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/2a064daf117a49e284fd0107acf6770c as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/2a064daf117a49e284fd0107acf6770c 2024-11-14T02:44:51,825 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 025639fc1f987c76e816bd52e91790c0/A of 025639fc1f987c76e816bd52e91790c0 into 2a064daf117a49e284fd0107acf6770c(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:51,825 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:44:51,825 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., storeName=025639fc1f987c76e816bd52e91790c0/A, priority=13, startTime=1731552291720; duration=0sec 2024-11-14T02:44:51,825 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:51,825 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 025639fc1f987c76e816bd52e91790c0:A 2024-11-14T02:44:51,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742145_1321 (size=12104) 2024-11-14T02:44:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T02:44:51,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T02:44:51,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:51,931 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 025639fc1f987c76e816bd52e91790c0 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-14T02:44:51,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143ef9ecea8894420580b8686225ef5017_025639fc1f987c76e816bd52e91790c0 is 50, key is test_row_0/A:col10/1731552290496/Put/seqid=0 2024-11-14T02:44:51,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742146_1322 (size=12154) 2024-11-14T02:44:52,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T02:44:52,235 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/8a6b3e767f2d40dc88a856971c266f27 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/8a6b3e767f2d40dc88a856971c266f27 2024-11-14T02:44:52,239 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 025639fc1f987c76e816bd52e91790c0/C of 025639fc1f987c76e816bd52e91790c0 into 8a6b3e767f2d40dc88a856971c266f27(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:52,240 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:44:52,240 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., storeName=025639fc1f987c76e816bd52e91790c0/C, priority=13, startTime=1731552291720; duration=0sec 2024-11-14T02:44:52,240 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:52,240 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 025639fc1f987c76e816bd52e91790c0:C 2024-11-14T02:44:52,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:52,359 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143ef9ecea8894420580b8686225ef5017_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143ef9ecea8894420580b8686225ef5017_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:52,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/8304a2838e9b46f58fb61d68b541bbd7, store: [table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:44:52,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/8304a2838e9b46f58fb61d68b541bbd7 is 175, key is test_row_0/A:col10/1731552290496/Put/seqid=0 2024-11-14T02:44:52,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742147_1323 (size=30955) 2024-11-14T02:44:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T02:44:52,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:52,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:52,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552352705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:52,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552352706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552352706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:52,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552352707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:52,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:52,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552352710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:52,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:52,799 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/8304a2838e9b46f58fb61d68b541bbd7 2024-11-14T02:44:52,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/ece360bf8f1343ec9c119d22994051b8 is 50, key is test_row_0/B:col10/1731552290496/Put/seqid=0 2024-11-14T02:44:52,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742148_1324 (size=12001) 2024-11-14T02:44:52,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552352814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552352814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:52,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552352814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:52,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552352823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552352823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:52,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T02:44:53,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:53,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552353023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:53,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:53,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:53,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552353023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:53,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:53,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:53,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552353023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:53,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:53,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:53,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552353033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:53,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:53,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:53,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552353034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:53,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:53,212 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/ece360bf8f1343ec9c119d22994051b8 2024-11-14T02:44:53,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/721cc64a63bb439db31802f3aacce79e is 50, key is test_row_0/C:col10/1731552290496/Put/seqid=0 2024-11-14T02:44:53,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742149_1325 (size=12001) 2024-11-14T02:44:53,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:53,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552353334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:53,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:53,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:53,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552353343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:53,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:53,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:53,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552353343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:53,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:53,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:53,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552353343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:53,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552353343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:53,628 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/721cc64a63bb439db31802f3aacce79e 2024-11-14T02:44:53,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/8304a2838e9b46f58fb61d68b541bbd7 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/8304a2838e9b46f58fb61d68b541bbd7 2024-11-14T02:44:53,635 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/8304a2838e9b46f58fb61d68b541bbd7, entries=150, sequenceid=79, filesize=30.2 K 2024-11-14T02:44:53,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/ece360bf8f1343ec9c119d22994051b8 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/ece360bf8f1343ec9c119d22994051b8 2024-11-14T02:44:53,639 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/ece360bf8f1343ec9c119d22994051b8, entries=150, sequenceid=79, filesize=11.7 K 2024-11-14T02:44:53,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/721cc64a63bb439db31802f3aacce79e as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/721cc64a63bb439db31802f3aacce79e 2024-11-14T02:44:53,645 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/721cc64a63bb439db31802f3aacce79e, entries=150, sequenceid=79, filesize=11.7 K 2024-11-14T02:44:53,646 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 025639fc1f987c76e816bd52e91790c0 in 1716ms, sequenceid=79, compaction requested=false 2024-11-14T02:44:53,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:44:53,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:53,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-14T02:44:53,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-14T02:44:53,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-14T02:44:53,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8700 sec 2024-11-14T02:44:53,652 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.8760 sec 2024-11-14T02:44:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:53,857 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:44:53,857 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 025639fc1f987c76e816bd52e91790c0 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-14T02:44:53,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111410abccd37e014168b540f2dd9391039c_025639fc1f987c76e816bd52e91790c0 is 50, key is test_row_0/A:col10/1731552292702/Put/seqid=0 2024-11-14T02:44:53,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742150_1326 (size=12154) 2024-11-14T02:44:53,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:53,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552353912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:53,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:53,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552353921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:53,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:53,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T02:44:53,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:53,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:53,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552353921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:53,923 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:53,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:53,923 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:53,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-14T02:44:53,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T02:44:53,925 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:53,925 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:53,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:53,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:53,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552353922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:53,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:53,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552353922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T02:44:54,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:54,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552354033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:54,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:54,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552354033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:54,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:54,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:54,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552354033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:54,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:54,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552354033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:54,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:54,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:54,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552354043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:54,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:54,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T02:44:54,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:54,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:54,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:54,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:54,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:54,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:54,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T02:44:54,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:54,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:54,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:54,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:54,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T02:44:54,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552354245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:54,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:54,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552354245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552354254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:54,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552354254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:54,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552354254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:54,296 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:54,299 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111410abccd37e014168b540f2dd9391039c_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111410abccd37e014168b540f2dd9391039c_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:54,300 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/0e9b42213e0040ab83a30f9b5c044c7f, store: [table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:44:54,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/0e9b42213e0040ab83a30f9b5c044c7f is 175, key is test_row_0/A:col10/1731552292702/Put/seqid=0 2024-11-14T02:44:54,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742151_1327 (size=30955) 2024-11-14T02:44:54,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T02:44:54,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:54,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:54,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:54,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:54,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:54,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:54,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T02:44:54,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:54,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:54,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:54,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:54,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T02:44:54,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552354563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:54,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:54,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552354563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552354563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552354563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:54,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552354563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:54,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T02:44:54,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:54,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:54,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:54,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:54,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:54,708 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/0e9b42213e0040ab83a30f9b5c044c7f 2024-11-14T02:44:54,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/bc30869f7f584e5cad1deff7bfe402fa is 50, key is test_row_0/B:col10/1731552292702/Put/seqid=0 2024-11-14T02:44:54,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742152_1328 (size=12001) 2024-11-14T02:44:54,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T02:44:54,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:54,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:54,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:54,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:54,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:54,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T02:44:54,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:54,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:54,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:54,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:54,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T02:44:55,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552355073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:55,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552355073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:55,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:55,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:55,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:55,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552355084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:55,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:55,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:55,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:55,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552355084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:55,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:55,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:55,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:55,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552355084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:55,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:55,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:55,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:55,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/bc30869f7f584e5cad1deff7bfe402fa 2024-11-14T02:44:55,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/3367fd6782c3484d93f6ab280c41703b is 50, key is test_row_0/C:col10/1731552292702/Put/seqid=0 2024-11-14T02:44:55,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742153_1329 (size=12001) 2024-11-14T02:44:55,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T02:44:55,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:55,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:55,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:55,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:55,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:55,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:55,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T02:44:55,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:55,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:55,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:55,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:55,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:55,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:55,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T02:44:55,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:55,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:55,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:55,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:55,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:55,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/3367fd6782c3484d93f6ab280c41703b 2024-11-14T02:44:55,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/0e9b42213e0040ab83a30f9b5c044c7f as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/0e9b42213e0040ab83a30f9b5c044c7f 2024-11-14T02:44:55,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/0e9b42213e0040ab83a30f9b5c044c7f, entries=150, sequenceid=94, filesize=30.2 K 2024-11-14T02:44:55,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/bc30869f7f584e5cad1deff7bfe402fa as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/bc30869f7f584e5cad1deff7bfe402fa 2024-11-14T02:44:55,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/bc30869f7f584e5cad1deff7bfe402fa, entries=150, sequenceid=94, filesize=11.7 K 2024-11-14T02:44:55,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/3367fd6782c3484d93f6ab280c41703b as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/3367fd6782c3484d93f6ab280c41703b 2024-11-14T02:44:55,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/3367fd6782c3484d93f6ab280c41703b, entries=150, sequenceid=94, filesize=11.7 K 2024-11-14T02:44:55,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 025639fc1f987c76e816bd52e91790c0 in 1686ms, sequenceid=94, compaction requested=true 2024-11-14T02:44:55,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:44:55,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 025639fc1f987c76e816bd52e91790c0:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:44:55,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:55,543 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:55,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 025639fc1f987c76e816bd52e91790c0:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:44:55,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:55,543 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:55,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 025639fc1f987c76e816bd52e91790c0:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:44:55,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:55,544 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:55,544 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 025639fc1f987c76e816bd52e91790c0/A is initiating minor compaction (all files) 2024-11-14T02:44:55,544 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 025639fc1f987c76e816bd52e91790c0/A in TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:55,545 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/2a064daf117a49e284fd0107acf6770c, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/8304a2838e9b46f58fb61d68b541bbd7, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/0e9b42213e0040ab83a30f9b5c044c7f] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp, totalSize=90.8 K 2024-11-14T02:44:55,545 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:55,545 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. files: [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/2a064daf117a49e284fd0107acf6770c, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/8304a2838e9b46f58fb61d68b541bbd7, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/0e9b42213e0040ab83a30f9b5c044c7f] 2024-11-14T02:44:55,550 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:55,550 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a064daf117a49e284fd0107acf6770c, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731552289836 2024-11-14T02:44:55,550 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 025639fc1f987c76e816bd52e91790c0/B is initiating minor compaction (all files) 2024-11-14T02:44:55,551 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 025639fc1f987c76e816bd52e91790c0/B in TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:55,551 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/e6064d0e2d394cf08c883deeb49b5317, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/ece360bf8f1343ec9c119d22994051b8, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/bc30869f7f584e5cad1deff7bfe402fa] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp, totalSize=35.3 K 2024-11-14T02:44:55,551 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8304a2838e9b46f58fb61d68b541bbd7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731552290496 2024-11-14T02:44:55,551 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting e6064d0e2d394cf08c883deeb49b5317, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731552289836 2024-11-14T02:44:55,551 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting ece360bf8f1343ec9c119d22994051b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731552290496 2024-11-14T02:44:55,551 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e9b42213e0040ab83a30f9b5c044c7f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731552292702 2024-11-14T02:44:55,551 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting bc30869f7f584e5cad1deff7bfe402fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731552292702 2024-11-14T02:44:55,556 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:44:55,558 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 025639fc1f987c76e816bd52e91790c0#B#compaction#278 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:55,558 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/8d2a0109c7354787b2c00050ee93556d is 50, key is test_row_0/B:col10/1731552292702/Put/seqid=0 2024-11-14T02:44:55,564 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114e3dde81abef54a18b5ff888686061fc8_025639fc1f987c76e816bd52e91790c0 store=[table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:44:55,566 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114e3dde81abef54a18b5ff888686061fc8_025639fc1f987c76e816bd52e91790c0, store=[table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:44:55,566 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e3dde81abef54a18b5ff888686061fc8_025639fc1f987c76e816bd52e91790c0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:44:55,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742154_1330 (size=12207) 2024-11-14T02:44:55,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742155_1331 (size=4469) 2024-11-14T02:44:55,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T02:44:55,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:55,606 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 025639fc1f987c76e816bd52e91790c0 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-14T02:44:55,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111474f94e5d304d425686d53a813ae3865d_025639fc1f987c76e816bd52e91790c0 is 50, key is test_row_0/A:col10/1731552293921/Put/seqid=0 2024-11-14T02:44:55,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742156_1332 (size=12154) 2024-11-14T02:44:55,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:55,622 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111474f94e5d304d425686d53a813ae3865d_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111474f94e5d304d425686d53a813ae3865d_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:55,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/4b0149e551534573bb20b9ac5b0ef911, store: [table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:44:55,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/4b0149e551534573bb20b9ac5b0ef911 is 175, key is test_row_0/A:col10/1731552293921/Put/seqid=0 2024-11-14T02:44:55,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742157_1333 (size=30955) 2024-11-14T02:44:55,979 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/8d2a0109c7354787b2c00050ee93556d as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/8d2a0109c7354787b2c00050ee93556d 2024-11-14T02:44:55,981 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 025639fc1f987c76e816bd52e91790c0#A#compaction#279 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:55,982 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/4daeeed6a61f40d48f03d9228be5e26a is 175, key is test_row_0/A:col10/1731552292702/Put/seqid=0 2024-11-14T02:44:55,983 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 025639fc1f987c76e816bd52e91790c0/B of 025639fc1f987c76e816bd52e91790c0 into 8d2a0109c7354787b2c00050ee93556d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:55,983 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:44:55,983 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., storeName=025639fc1f987c76e816bd52e91790c0/B, priority=13, startTime=1731552295543; duration=0sec 2024-11-14T02:44:55,984 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:55,984 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 025639fc1f987c76e816bd52e91790c0:B 2024-11-14T02:44:55,984 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:55,985 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:55,985 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 025639fc1f987c76e816bd52e91790c0/C is initiating minor compaction (all files) 2024-11-14T02:44:55,985 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 025639fc1f987c76e816bd52e91790c0/C in TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:55,985 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/8a6b3e767f2d40dc88a856971c266f27, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/721cc64a63bb439db31802f3aacce79e, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/3367fd6782c3484d93f6ab280c41703b] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp, totalSize=35.3 K 2024-11-14T02:44:55,985 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a6b3e767f2d40dc88a856971c266f27, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731552289836 2024-11-14T02:44:55,985 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 721cc64a63bb439db31802f3aacce79e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731552290496 2024-11-14T02:44:55,986 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 3367fd6782c3484d93f6ab280c41703b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731552292702 2024-11-14T02:44:55,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742158_1334 (size=31161) 2024-11-14T02:44:55,994 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 025639fc1f987c76e816bd52e91790c0#C#compaction#281 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:55,995 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/a74c88847781457ca02dd3f3d2f4ca68 is 50, key is test_row_0/C:col10/1731552292702/Put/seqid=0 2024-11-14T02:44:55,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742159_1335 (size=12207) 2024-11-14T02:44:56,028 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/4b0149e551534573bb20b9ac5b0ef911 2024-11-14T02:44:56,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/3576ee5713e2465a93d119326bebc21b is 50, key is test_row_0/B:col10/1731552293921/Put/seqid=0 2024-11-14T02:44:56,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742160_1336 (size=12001) 2024-11-14T02:44:56,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T02:44:56,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:56,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:56,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:56,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552356110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552356110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:56,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552356110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552356113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:56,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:56,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552356114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:56,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:56,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552356223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:56,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:56,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552356223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:56,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:56,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:56,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:56,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552356223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:56,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:56,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552356223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:56,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:56,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552356234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:56,390 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/4daeeed6a61f40d48f03d9228be5e26a as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/4daeeed6a61f40d48f03d9228be5e26a 2024-11-14T02:44:56,394 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 025639fc1f987c76e816bd52e91790c0/A of 025639fc1f987c76e816bd52e91790c0 into 4daeeed6a61f40d48f03d9228be5e26a(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:56,394 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:44:56,394 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., storeName=025639fc1f987c76e816bd52e91790c0/A, priority=13, startTime=1731552295543; duration=0sec 2024-11-14T02:44:56,394 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:56,394 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 025639fc1f987c76e816bd52e91790c0:A 2024-11-14T02:44:56,401 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/a74c88847781457ca02dd3f3d2f4ca68 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/a74c88847781457ca02dd3f3d2f4ca68 2024-11-14T02:44:56,404 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 025639fc1f987c76e816bd52e91790c0/C of 025639fc1f987c76e816bd52e91790c0 into a74c88847781457ca02dd3f3d2f4ca68(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:56,404 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:44:56,404 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., storeName=025639fc1f987c76e816bd52e91790c0/C, priority=13, startTime=1731552295543; duration=0sec 2024-11-14T02:44:56,405 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:56,405 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 025639fc1f987c76e816bd52e91790c0:C 2024-11-14T02:44:56,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:56,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552356433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:56,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:56,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552356433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:56,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:56,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:56,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552356433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:56,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:56,440 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/3576ee5713e2465a93d119326bebc21b 2024-11-14T02:44:56,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:56,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/d9f7916095b940518bc869ed1eb75787 is 50, key is test_row_0/C:col10/1731552293921/Put/seqid=0 2024-11-14T02:44:56,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552356444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:56,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:56,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742161_1337 (size=12001) 2024-11-14T02:44:56,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552356444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:56,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:56,451 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/d9f7916095b940518bc869ed1eb75787 2024-11-14T02:44:56,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/4b0149e551534573bb20b9ac5b0ef911 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/4b0149e551534573bb20b9ac5b0ef911 2024-11-14T02:44:56,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:56,460 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/4b0149e551534573bb20b9ac5b0ef911, entries=150, sequenceid=117, filesize=30.2 K 2024-11-14T02:44:56,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/3576ee5713e2465a93d119326bebc21b as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/3576ee5713e2465a93d119326bebc21b 2024-11-14T02:44:56,467 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/3576ee5713e2465a93d119326bebc21b, entries=150, sequenceid=117, filesize=11.7 K 2024-11-14T02:44:56,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/d9f7916095b940518bc869ed1eb75787 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/d9f7916095b940518bc869ed1eb75787 2024-11-14T02:44:56,471 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/d9f7916095b940518bc869ed1eb75787, entries=150, sequenceid=117, filesize=11.7 K 2024-11-14T02:44:56,472 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 025639fc1f987c76e816bd52e91790c0 in 866ms, sequenceid=117, compaction requested=false 2024-11-14T02:44:56,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:44:56,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:56,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-14T02:44:56,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-14T02:44:56,475 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-14T02:44:56,475 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5490 sec 2024-11-14T02:44:56,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.5520 sec 2024-11-14T02:44:56,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:56,748 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:44:56,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 025639fc1f987c76e816bd52e91790c0 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-14T02:44:56,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fdf189c54b1740ab9fcfd8b08607802e_025639fc1f987c76e816bd52e91790c0 is 50, key is test_row_0/A:col10/1731552296109/Put/seqid=0 2024-11-14T02:44:56,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742162_1338 (size=12254) 2024-11-14T02:44:56,772 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:56,775 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fdf189c54b1740ab9fcfd8b08607802e_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fdf189c54b1740ab9fcfd8b08607802e_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:56,776 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/5544312e1ee1418f8d0246a089b09e7b, store: [table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:44:56,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/5544312e1ee1418f8d0246a089b09e7b is 175, key is test_row_0/A:col10/1731552296109/Put/seqid=0 2024-11-14T02:44:56,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742163_1339 (size=31055) 2024-11-14T02:44:56,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552356785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:56,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:56,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552356786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:56,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552356786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:56,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552356789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:56,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552356790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:56,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:56,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552356893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:56,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:56,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:56,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552356903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:56,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:56,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:56,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552356903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:56,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:56,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552356903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:56,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552356903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:57,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552357103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:57,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:57,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552357113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:57,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:57,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552357113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:57,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:57,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552357123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:57,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:57,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552357124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:57,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:57,180 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/5544312e1ee1418f8d0246a089b09e7b 2024-11-14T02:44:57,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/c510a687d228429ab916a5716030f26e is 50, key is test_row_0/B:col10/1731552296109/Put/seqid=0 2024-11-14T02:44:57,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742164_1340 (size=12101) 2024-11-14T02:44:57,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552357413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:57,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:57,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552357423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:57,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552357423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:57,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552357443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:57,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:57,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552357443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:57,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:57,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/c510a687d228429ab916a5716030f26e 2024-11-14T02:44:57,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/5c246bb8ca424d48b37f4136ee50da97 is 50, key is test_row_0/C:col10/1731552296109/Put/seqid=0 2024-11-14T02:44:57,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742165_1341 (size=12101) 2024-11-14T02:44:57,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/5c246bb8ca424d48b37f4136ee50da97 2024-11-14T02:44:57,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/5544312e1ee1418f8d0246a089b09e7b as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/5544312e1ee1418f8d0246a089b09e7b 2024-11-14T02:44:57,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/5544312e1ee1418f8d0246a089b09e7b, entries=150, sequenceid=135, filesize=30.3 K 2024-11-14T02:44:57,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/c510a687d228429ab916a5716030f26e as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/c510a687d228429ab916a5716030f26e 2024-11-14T02:44:57,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/c510a687d228429ab916a5716030f26e, entries=150, sequenceid=135, filesize=11.8 K 2024-11-14T02:44:57,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/5c246bb8ca424d48b37f4136ee50da97 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/5c246bb8ca424d48b37f4136ee50da97 2024-11-14T02:44:57,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/5c246bb8ca424d48b37f4136ee50da97, entries=150, sequenceid=135, filesize=11.8 K 2024-11-14T02:44:57,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 025639fc1f987c76e816bd52e91790c0 in 894ms, sequenceid=135, compaction requested=true 2024-11-14T02:44:57,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:44:57,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 025639fc1f987c76e816bd52e91790c0:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:44:57,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:57,643 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:57,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 025639fc1f987c76e816bd52e91790c0:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:44:57,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:57,643 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:57,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 025639fc1f987c76e816bd52e91790c0:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:44:57,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:57,643 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:57,643 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93171 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:57,643 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 025639fc1f987c76e816bd52e91790c0/B is initiating minor compaction (all files) 2024-11-14T02:44:57,643 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 025639fc1f987c76e816bd52e91790c0/A is initiating minor compaction (all files) 2024-11-14T02:44:57,643 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 025639fc1f987c76e816bd52e91790c0/B in TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:57,643 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 025639fc1f987c76e816bd52e91790c0/A in TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:57,643 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/8d2a0109c7354787b2c00050ee93556d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/3576ee5713e2465a93d119326bebc21b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/c510a687d228429ab916a5716030f26e] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp, totalSize=35.5 K 2024-11-14T02:44:57,644 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/4daeeed6a61f40d48f03d9228be5e26a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/4b0149e551534573bb20b9ac5b0ef911, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/5544312e1ee1418f8d0246a089b09e7b] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp, totalSize=91.0 K 2024-11-14T02:44:57,644 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:57,644 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. files: [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/4daeeed6a61f40d48f03d9228be5e26a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/4b0149e551534573bb20b9ac5b0ef911, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/5544312e1ee1418f8d0246a089b09e7b] 2024-11-14T02:44:57,644 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d2a0109c7354787b2c00050ee93556d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731552292702 2024-11-14T02:44:57,644 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4daeeed6a61f40d48f03d9228be5e26a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731552292702 2024-11-14T02:44:57,644 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 3576ee5713e2465a93d119326bebc21b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731552293911 2024-11-14T02:44:57,644 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b0149e551534573bb20b9ac5b0ef911, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731552293911 2024-11-14T02:44:57,644 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting c510a687d228429ab916a5716030f26e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731552296109 2024-11-14T02:44:57,644 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5544312e1ee1418f8d0246a089b09e7b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731552296109 2024-11-14T02:44:57,651 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 025639fc1f987c76e816bd52e91790c0#B#compaction#287 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:57,651 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/b5dc534315434f8ea75ce2a25f740b71 is 50, key is test_row_0/B:col10/1731552296109/Put/seqid=0 2024-11-14T02:44:57,652 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:44:57,659 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111431a2fbfb5d7d4a739bf73cabc1521bb0_025639fc1f987c76e816bd52e91790c0 store=[table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:44:57,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742166_1342 (size=12409) 2024-11-14T02:44:57,662 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111431a2fbfb5d7d4a739bf73cabc1521bb0_025639fc1f987c76e816bd52e91790c0, store=[table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:44:57,662 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111431a2fbfb5d7d4a739bf73cabc1521bb0_025639fc1f987c76e816bd52e91790c0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:44:57,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742167_1343 (size=4469) 2024-11-14T02:44:57,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:57,929 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:44:57,929 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 025639fc1f987c76e816bd52e91790c0 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-14T02:44:57,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a969a3d2ba8749d88005780e9f7ca68c_025639fc1f987c76e816bd52e91790c0 is 50, key is test_row_0/A:col10/1731552296789/Put/seqid=0 2024-11-14T02:44:57,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742168_1344 (size=14794) 2024-11-14T02:44:57,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:57,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552357954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:57,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:57,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:57,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552357956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:57,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:57,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:57,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552357959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:57,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:57,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:57,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552357960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:57,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:57,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:57,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552357963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:57,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:57,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:58,065 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/b5dc534315434f8ea75ce2a25f740b71 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/b5dc534315434f8ea75ce2a25f740b71 2024-11-14T02:44:58,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:58,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552358063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:58,069 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 025639fc1f987c76e816bd52e91790c0/B of 025639fc1f987c76e816bd52e91790c0 into b5dc534315434f8ea75ce2a25f740b71(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:58,069 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:44:58,069 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., storeName=025639fc1f987c76e816bd52e91790c0/B, priority=13, startTime=1731552297643; duration=0sec 2024-11-14T02:44:58,069 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:58,069 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 025639fc1f987c76e816bd52e91790c0:B 2024-11-14T02:44:58,069 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:58,070 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:58,070 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 025639fc1f987c76e816bd52e91790c0/C is initiating minor compaction (all files) 2024-11-14T02:44:58,070 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 025639fc1f987c76e816bd52e91790c0/C in TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:58,070 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/a74c88847781457ca02dd3f3d2f4ca68, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/d9f7916095b940518bc869ed1eb75787, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/5c246bb8ca424d48b37f4136ee50da97] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp, totalSize=35.5 K 2024-11-14T02:44:58,070 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting a74c88847781457ca02dd3f3d2f4ca68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731552292702 2024-11-14T02:44:58,071 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting d9f7916095b940518bc869ed1eb75787, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731552293911 2024-11-14T02:44:58,071 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c246bb8ca424d48b37f4136ee50da97, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731552296109 2024-11-14T02:44:58,076 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 025639fc1f987c76e816bd52e91790c0#A#compaction#288 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:58,076 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/cdd69051d77d40bea077532de86335f4 is 175, key is test_row_0/A:col10/1731552296109/Put/seqid=0 2024-11-14T02:44:58,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:58,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552358073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:58,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:58,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:58,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:58,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742169_1345 (size=31363) 2024-11-14T02:44:58,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:58,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:58,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552358075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:58,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552358075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:58,085 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 025639fc1f987c76e816bd52e91790c0#C#compaction#290 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:58,085 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/8f392c164c934580bbd99442f961be67 is 50, key is test_row_0/C:col10/1731552296109/Put/seqid=0 2024-11-14T02:44:58,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742170_1346 (size=12409) 2024-11-14T02:44:58,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T02:44:58,094 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:58,094 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:58,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-14T02:44:58,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T02:44:58,096 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:58,096 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:58,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:58,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T02:44:58,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T02:44:58,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:58,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:58,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:58,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:58,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:58,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:58,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552358273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:58,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:58,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:58,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:58,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552358283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:58,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:58,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:58,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:58,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:58,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552358294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:58,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:58,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:58,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552358294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:58,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:58,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:58,347 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:58,350 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a969a3d2ba8749d88005780e9f7ca68c_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a969a3d2ba8749d88005780e9f7ca68c_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:44:58,350 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/503e708177a84275b0d6125fe01463a4, store: [table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:44:58,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/503e708177a84275b0d6125fe01463a4 is 175, key is test_row_0/A:col10/1731552296789/Put/seqid=0 2024-11-14T02:44:58,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742171_1347 (size=39749) 2024-11-14T02:44:58,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T02:44:58,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:58,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:58,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:58,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:58,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:58,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:58,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T02:44:58,489 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/cdd69051d77d40bea077532de86335f4 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/cdd69051d77d40bea077532de86335f4 2024-11-14T02:44:58,492 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/8f392c164c934580bbd99442f961be67 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/8f392c164c934580bbd99442f961be67 2024-11-14T02:44:58,494 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 025639fc1f987c76e816bd52e91790c0/A of 025639fc1f987c76e816bd52e91790c0 into cdd69051d77d40bea077532de86335f4(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:58,494 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:44:58,494 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., storeName=025639fc1f987c76e816bd52e91790c0/A, priority=13, startTime=1731552297642; duration=0sec 2024-11-14T02:44:58,495 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:58,495 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 025639fc1f987c76e816bd52e91790c0:A 2024-11-14T02:44:58,497 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 025639fc1f987c76e816bd52e91790c0/C of 025639fc1f987c76e816bd52e91790c0 into 8f392c164c934580bbd99442f961be67(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:58,497 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:44:58,497 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., storeName=025639fc1f987c76e816bd52e91790c0/C, priority=13, startTime=1731552297643; duration=0sec 2024-11-14T02:44:58,497 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:58,497 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 025639fc1f987c76e816bd52e91790c0:C 2024-11-14T02:44:58,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T02:44:58,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:58,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:58,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:58,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:58,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:58,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:58,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552358583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:58,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:58,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552358593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:58,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:58,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:58,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:58,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:58,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552358603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:58,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:58,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:58,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552358614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:58,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:58,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:58,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:58,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T02:44:58,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:58,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:58,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:58,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:58,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:58,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T02:44:58,755 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/503e708177a84275b0d6125fe01463a4 2024-11-14T02:44:58,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/0e97659e64f7455ebb8299db7837a274 is 50, key is test_row_0/B:col10/1731552296789/Put/seqid=0 2024-11-14T02:44:58,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742172_1348 (size=12151) 2024-11-14T02:44:58,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T02:44:58,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:58,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:58,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:58,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:58,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:58,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:58,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:58,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552358983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:58,989 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_0' in 025639fc1f987c76e816bd52e91790c0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:44:58,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:58,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:58,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:59,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T02:44:59,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:59,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:59,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:59,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:59,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:59,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:59,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:59,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552359093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:59,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:59,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552359103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:59,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:59,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:59,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:59,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:59,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552359113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:59,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:59,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:59,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:59,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:44:59,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552359133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:44:59,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:59,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:59,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:59,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T02:44:59,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:59,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:59,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:59,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:59,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:59,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/0e97659e64f7455ebb8299db7837a274 2024-11-14T02:44:59,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/1ed5c1e865684bc0b85aa807b5c45168 is 50, key is test_row_0/C:col10/1731552296789/Put/seqid=0 2024-11-14T02:44:59,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742173_1349 (size=12151) 2024-11-14T02:44:59,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T02:44:59,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T02:44:59,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:59,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:59,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:59,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:59,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:59,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T02:44:59,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:59,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:44:59,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:59,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:59,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:59,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:59,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/1ed5c1e865684bc0b85aa807b5c45168 2024-11-14T02:44:59,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/503e708177a84275b0d6125fe01463a4 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/503e708177a84275b0d6125fe01463a4 2024-11-14T02:44:59,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/503e708177a84275b0d6125fe01463a4, entries=200, sequenceid=157, filesize=38.8 K 2024-11-14T02:44:59,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/0e97659e64f7455ebb8299db7837a274 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/0e97659e64f7455ebb8299db7837a274 2024-11-14T02:44:59,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/0e97659e64f7455ebb8299db7837a274, entries=150, sequenceid=157, filesize=11.9 K 2024-11-14T02:44:59,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/1ed5c1e865684bc0b85aa807b5c45168 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/1ed5c1e865684bc0b85aa807b5c45168 2024-11-14T02:44:59,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/1ed5c1e865684bc0b85aa807b5c45168, entries=150, sequenceid=157, filesize=11.9 K 2024-11-14T02:44:59,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 025639fc1f987c76e816bd52e91790c0 in 1663ms, sequenceid=157, compaction requested=false 2024-11-14T02:44:59,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:44:59,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T02:44:59,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:44:59,622 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 025639fc1f987c76e816bd52e91790c0 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-14T02:44:59,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f31cd3547af4491e9241e4eabc0554ff_025639fc1f987c76e816bd52e91790c0 is 50, key is test_row_0/A:col10/1731552297959/Put/seqid=0 2024-11-14T02:44:59,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742174_1350 (size=12304) 2024-11-14T02:45:00,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:00,035 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f31cd3547af4491e9241e4eabc0554ff_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f31cd3547af4491e9241e4eabc0554ff_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:00,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/cfe2722f60f74560b77649a05ad0ea55, store: [table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:45:00,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/cfe2722f60f74560b77649a05ad0ea55 is 175, key is test_row_0/A:col10/1731552297959/Put/seqid=0 2024-11-14T02:45:00,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742175_1351 (size=31105) 2024-11-14T02:45:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:00,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:45:00,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:00,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552360158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:00,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:00,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:00,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552360167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:00,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:00,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552360168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:00,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552360169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:00,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T02:45:00,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:00,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552360273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:00,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:00,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552360273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:00,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:00,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:00,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:00,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552360283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:00,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:00,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:00,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552360283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:00,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:00,445 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/cfe2722f60f74560b77649a05ad0ea55 2024-11-14T02:45:00,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/4f1b4683d1f7420ba5aa3550db4158ec is 50, key is test_row_0/B:col10/1731552297959/Put/seqid=0 2024-11-14T02:45:00,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742176_1352 (size=12151) 2024-11-14T02:45:00,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552360483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:00,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:00,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552360483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:00,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:00,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:00,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552360494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:00,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:00,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:00,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552360494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:00,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:00,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552360793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:00,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552360793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:00,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:00,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552360803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:00,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:00,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552360803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:00,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:00,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:00,857 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/4f1b4683d1f7420ba5aa3550db4158ec 2024-11-14T02:45:00,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/43fc37146cd14d9da86bfdca4636a314 is 50, key is test_row_0/C:col10/1731552297959/Put/seqid=0 2024-11-14T02:45:00,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742177_1353 (size=12151) 2024-11-14T02:45:01,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552361014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:01,020 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_0' in 025639fc1f987c76e816bd52e91790c0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:45:01,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:01,266 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/43fc37146cd14d9da86bfdca4636a314 2024-11-14T02:45:01,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/cfe2722f60f74560b77649a05ad0ea55 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/cfe2722f60f74560b77649a05ad0ea55 2024-11-14T02:45:01,272 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/cfe2722f60f74560b77649a05ad0ea55, entries=150, sequenceid=174, filesize=30.4 K 2024-11-14T02:45:01,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/4f1b4683d1f7420ba5aa3550db4158ec as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/4f1b4683d1f7420ba5aa3550db4158ec 2024-11-14T02:45:01,276 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/4f1b4683d1f7420ba5aa3550db4158ec, entries=150, sequenceid=174, filesize=11.9 K 2024-11-14T02:45:01,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/43fc37146cd14d9da86bfdca4636a314 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/43fc37146cd14d9da86bfdca4636a314 2024-11-14T02:45:01,280 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/43fc37146cd14d9da86bfdca4636a314, entries=150, sequenceid=174, filesize=11.9 K 2024-11-14T02:45:01,280 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 025639fc1f987c76e816bd52e91790c0 in 1658ms, sequenceid=174, compaction requested=true 2024-11-14T02:45:01,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:45:01,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:01,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-14T02:45:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-14T02:45:01,283 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-14T02:45:01,283 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1860 sec 2024-11-14T02:45:01,285 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 3.1900 sec 2024-11-14T02:45:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:01,309 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:45:01,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 025639fc1f987c76e816bd52e91790c0 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-14T02:45:01,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143ad1e9996d5542dda5b33a3d5bc1c8a7_025639fc1f987c76e816bd52e91790c0 is 50, key is test_row_0/A:col10/1731552300167/Put/seqid=0 2024-11-14T02:45:01,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742178_1354 (size=14794) 2024-11-14T02:45:01,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552361330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:01,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:01,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:01,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552361335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:01,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:01,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:01,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552361335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:01,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:01,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:01,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552361336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:01,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:01,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552361443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552361443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552361444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:01,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:01,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552361453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:01,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:01,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552361653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:01,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:01,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552361653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:01,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:01,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:01,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552361663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:01,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552361664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:01,720 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:01,723 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143ad1e9996d5542dda5b33a3d5bc1c8a7_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143ad1e9996d5542dda5b33a3d5bc1c8a7_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:01,724 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/d1d341a514bc45058a0676665d38a2fa, store: [table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:45:01,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/d1d341a514bc45058a0676665d38a2fa is 175, key is test_row_0/A:col10/1731552300167/Put/seqid=0 2024-11-14T02:45:01,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742179_1355 (size=39749) 2024-11-14T02:45:01,729 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/d1d341a514bc45058a0676665d38a2fa 2024-11-14T02:45:01,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/d22ee03f86ec485e972f08ef41485755 is 50, key is test_row_0/B:col10/1731552300167/Put/seqid=0 2024-11-14T02:45:01,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742180_1356 (size=12151) 2024-11-14T02:45:01,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:01,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552361973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:01,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:01,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:01,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552361973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:01,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:01,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:01,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552361973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:01,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:01,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:01,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552361973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:01,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:01,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:02,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/d22ee03f86ec485e972f08ef41485755 2024-11-14T02:45:02,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/feaeb18f657e463e9aaa14b6fd036262 is 50, key is test_row_0/C:col10/1731552300167/Put/seqid=0 2024-11-14T02:45:02,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742181_1357 (size=12151) 2024-11-14T02:45:02,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T02:45:02,264 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:02,264 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:02,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-14T02:45:02,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T02:45:02,266 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:02,266 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:02,266 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T02:45:02,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T02:45:02,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:02,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:45:02,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:02,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:02,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:02,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:02,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:02,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552362483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:02,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:02,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552362484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:02,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:02,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552362484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:02,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:02,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552362484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:02,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:02,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:02,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:02,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/feaeb18f657e463e9aaa14b6fd036262 2024-11-14T02:45:02,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/d1d341a514bc45058a0676665d38a2fa as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d1d341a514bc45058a0676665d38a2fa 2024-11-14T02:45:02,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d1d341a514bc45058a0676665d38a2fa, entries=200, sequenceid=198, filesize=38.8 K 2024-11-14T02:45:02,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/d22ee03f86ec485e972f08ef41485755 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/d22ee03f86ec485e972f08ef41485755 2024-11-14T02:45:02,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/d22ee03f86ec485e972f08ef41485755, entries=150, sequenceid=198, filesize=11.9 K 2024-11-14T02:45:02,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/feaeb18f657e463e9aaa14b6fd036262 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/feaeb18f657e463e9aaa14b6fd036262 2024-11-14T02:45:02,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/feaeb18f657e463e9aaa14b6fd036262, entries=150, sequenceid=198, filesize=11.9 K 2024-11-14T02:45:02,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 025639fc1f987c76e816bd52e91790c0 in 1255ms, sequenceid=198, compaction requested=true 2024-11-14T02:45:02,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:45:02,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 025639fc1f987c76e816bd52e91790c0:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:45:02,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:02,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 025639fc1f987c76e816bd52e91790c0:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:45:02,565 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:45:02,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:02,565 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:45:02,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 025639fc1f987c76e816bd52e91790c0:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:45:02,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:02,566 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:45:02,566 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141966 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:45:02,566 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 025639fc1f987c76e816bd52e91790c0/A is initiating minor compaction (all files) 2024-11-14T02:45:02,566 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 025639fc1f987c76e816bd52e91790c0/B is initiating minor compaction (all files) 2024-11-14T02:45:02,566 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 025639fc1f987c76e816bd52e91790c0/B in TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:02,566 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 025639fc1f987c76e816bd52e91790c0/A in TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:02,566 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/b5dc534315434f8ea75ce2a25f740b71, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/0e97659e64f7455ebb8299db7837a274, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/4f1b4683d1f7420ba5aa3550db4158ec, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/d22ee03f86ec485e972f08ef41485755] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp, totalSize=47.7 K 2024-11-14T02:45:02,566 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/cdd69051d77d40bea077532de86335f4, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/503e708177a84275b0d6125fe01463a4, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/cfe2722f60f74560b77649a05ad0ea55, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d1d341a514bc45058a0676665d38a2fa] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp, totalSize=138.6 K 2024-11-14T02:45:02,566 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:02,566 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. files: [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/cdd69051d77d40bea077532de86335f4, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/503e708177a84275b0d6125fe01463a4, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/cfe2722f60f74560b77649a05ad0ea55, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d1d341a514bc45058a0676665d38a2fa] 2024-11-14T02:45:02,566 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting b5dc534315434f8ea75ce2a25f740b71, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731552296109 2024-11-14T02:45:02,566 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdd69051d77d40bea077532de86335f4, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731552296109 2024-11-14T02:45:02,567 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 503e708177a84275b0d6125fe01463a4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731552296771 2024-11-14T02:45:02,567 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e97659e64f7455ebb8299db7837a274, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731552296771 2024-11-14T02:45:02,567 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f1b4683d1f7420ba5aa3550db4158ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731552297955 2024-11-14T02:45:02,567 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfe2722f60f74560b77649a05ad0ea55, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731552297955 2024-11-14T02:45:02,567 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1d341a514bc45058a0676665d38a2fa, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731552300166 2024-11-14T02:45:02,567 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting d22ee03f86ec485e972f08ef41485755, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731552300166 2024-11-14T02:45:02,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T02:45:02,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:02,572 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 025639fc1f987c76e816bd52e91790c0 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T02:45:02,574 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 025639fc1f987c76e816bd52e91790c0#B#compaction#299 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:02,575 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/cfab71f76a3f491d857991e9985f413c is 50, key is test_row_0/B:col10/1731552300167/Put/seqid=0 2024-11-14T02:45:02,576 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:45:02,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T02:45:02,584 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114319e21257ebc40b083cae33a1c3e4012_025639fc1f987c76e816bd52e91790c0 store=[table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:45:02,586 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114319e21257ebc40b083cae33a1c3e4012_025639fc1f987c76e816bd52e91790c0, store=[table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:45:02,586 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114319e21257ebc40b083cae33a1c3e4012_025639fc1f987c76e816bd52e91790c0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:45:02,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144847412cd2d24f4ea6bcd9b04c65b5a9_025639fc1f987c76e816bd52e91790c0 is 50, key is test_row_0/A:col10/1731552301334/Put/seqid=0 2024-11-14T02:45:02,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742182_1358 (size=12595) 2024-11-14T02:45:02,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742183_1359 (size=4469) 2024-11-14T02:45:02,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742184_1360 (size=12304) 2024-11-14T02:45:02,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:02,633 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144847412cd2d24f4ea6bcd9b04c65b5a9_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144847412cd2d24f4ea6bcd9b04c65b5a9_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:02,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/d78dca422675490b9bd2ea68119c1088, store: [table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:45:02,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/d78dca422675490b9bd2ea68119c1088 is 175, key is test_row_0/A:col10/1731552301334/Put/seqid=0 2024-11-14T02:45:02,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742185_1361 (size=31105) 2024-11-14T02:45:02,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T02:45:03,023 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 025639fc1f987c76e816bd52e91790c0#A#compaction#300 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:03,024 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/ebe60dac90404f7587358250cd916d84 is 175, key is test_row_0/A:col10/1731552300167/Put/seqid=0 2024-11-14T02:45:03,025 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/cfab71f76a3f491d857991e9985f413c as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/cfab71f76a3f491d857991e9985f413c 2024-11-14T02:45:03,030 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 025639fc1f987c76e816bd52e91790c0/B of 025639fc1f987c76e816bd52e91790c0 into cfab71f76a3f491d857991e9985f413c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:03,030 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:45:03,030 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., storeName=025639fc1f987c76e816bd52e91790c0/B, priority=12, startTime=1731552302565; duration=0sec 2024-11-14T02:45:03,030 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:03,030 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 025639fc1f987c76e816bd52e91790c0:B 2024-11-14T02:45:03,030 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:45:03,032 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:45:03,032 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 025639fc1f987c76e816bd52e91790c0/C is initiating minor compaction (all files) 2024-11-14T02:45:03,032 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 025639fc1f987c76e816bd52e91790c0/C in TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:03,032 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/8f392c164c934580bbd99442f961be67, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/1ed5c1e865684bc0b85aa807b5c45168, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/43fc37146cd14d9da86bfdca4636a314, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/feaeb18f657e463e9aaa14b6fd036262] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp, totalSize=47.7 K 2024-11-14T02:45:03,032 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f392c164c934580bbd99442f961be67, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731552296109 2024-11-14T02:45:03,032 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ed5c1e865684bc0b85aa807b5c45168, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731552296771 2024-11-14T02:45:03,033 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 43fc37146cd14d9da86bfdca4636a314, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731552297955 2024-11-14T02:45:03,033 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting feaeb18f657e463e9aaa14b6fd036262, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731552300166 2024-11-14T02:45:03,038 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/d78dca422675490b9bd2ea68119c1088 2024-11-14T02:45:03,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742186_1362 (size=31549) 2024-11-14T02:45:03,042 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 025639fc1f987c76e816bd52e91790c0#C#compaction#302 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:03,042 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/fffe715202fe439cb29f7ab2de5d9b21 is 50, key is test_row_0/C:col10/1731552300167/Put/seqid=0 2024-11-14T02:45:03,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/40b7e9d05d16406c94522577523e92f2 is 50, key is test_row_0/B:col10/1731552301334/Put/seqid=0 2024-11-14T02:45:03,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742187_1363 (size=12595) 2024-11-14T02:45:03,050 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/ebe60dac90404f7587358250cd916d84 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/ebe60dac90404f7587358250cd916d84 2024-11-14T02:45:03,056 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/fffe715202fe439cb29f7ab2de5d9b21 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/fffe715202fe439cb29f7ab2de5d9b21 2024-11-14T02:45:03,057 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 025639fc1f987c76e816bd52e91790c0/A of 025639fc1f987c76e816bd52e91790c0 into ebe60dac90404f7587358250cd916d84(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:03,057 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:45:03,057 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., storeName=025639fc1f987c76e816bd52e91790c0/A, priority=12, startTime=1731552302565; duration=0sec 2024-11-14T02:45:03,057 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:03,057 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 025639fc1f987c76e816bd52e91790c0:A 2024-11-14T02:45:03,063 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 025639fc1f987c76e816bd52e91790c0/C of 025639fc1f987c76e816bd52e91790c0 into fffe715202fe439cb29f7ab2de5d9b21(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:03,063 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:45:03,063 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., storeName=025639fc1f987c76e816bd52e91790c0/C, priority=12, startTime=1731552302565; duration=0sec 2024-11-14T02:45:03,063 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:03,063 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 025639fc1f987c76e816bd52e91790c0:C 2024-11-14T02:45:03,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742188_1364 (size=12151) 2024-11-14T02:45:03,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T02:45:03,465 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/40b7e9d05d16406c94522577523e92f2 2024-11-14T02:45:03,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/92c4752f1c364b0caefa5e6e53cc2254 is 50, key is test_row_0/C:col10/1731552301334/Put/seqid=0 2024-11-14T02:45:03,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742189_1365 (size=12151) 2024-11-14T02:45:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:03,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:45:03,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552363536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:03,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:03,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:03,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:03,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:03,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552363543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:03,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:03,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:03,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:03,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:03,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552363544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:03,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:03,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:03,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:03,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:03,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552363545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:03,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:03,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:03,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:03,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552363653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:03,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:03,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552363653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:03,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:03,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:03,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:03,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:03,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:03,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552363663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:03,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552363663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:03,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552363863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:03,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:03,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552363863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:03,874 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/92c4752f1c364b0caefa5e6e53cc2254 2024-11-14T02:45:03,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/d78dca422675490b9bd2ea68119c1088 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d78dca422675490b9bd2ea68119c1088 2024-11-14T02:45:03,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:03,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552363874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:03,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:03,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552363874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:03,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:03,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:03,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:03,882 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d78dca422675490b9bd2ea68119c1088, entries=150, sequenceid=210, filesize=30.4 K 2024-11-14T02:45:03,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/40b7e9d05d16406c94522577523e92f2 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/40b7e9d05d16406c94522577523e92f2 2024-11-14T02:45:03,887 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/40b7e9d05d16406c94522577523e92f2, entries=150, sequenceid=210, filesize=11.9 K 2024-11-14T02:45:03,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/92c4752f1c364b0caefa5e6e53cc2254 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/92c4752f1c364b0caefa5e6e53cc2254 2024-11-14T02:45:03,891 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/92c4752f1c364b0caefa5e6e53cc2254, entries=150, sequenceid=210, filesize=11.9 K 2024-11-14T02:45:03,893 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 025639fc1f987c76e816bd52e91790c0 in 1321ms, sequenceid=210, compaction requested=false 2024-11-14T02:45:03,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:45:03,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:03,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-14T02:45:03,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-14T02:45:03,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-14T02:45:03,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6290 sec 2024-11-14T02:45:03,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.6330 sec 2024-11-14T02:45:04,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:04,176 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:45:04,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 025639fc1f987c76e816bd52e91790c0 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T02:45:04,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144d49e1b97483458b9215f8eae6535f64_025639fc1f987c76e816bd52e91790c0 is 50, key is test_row_0/A:col10/1731552304175/Put/seqid=0 2024-11-14T02:45:04,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742190_1366 (size=14794) 2024-11-14T02:45:04,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:04,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552364224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:04,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552364225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:04,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:04,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552364226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:04,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:04,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552364227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:04,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:04,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:04,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552364344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:04,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:04,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552364344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:04,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:04,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:04,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552364344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:04,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:04,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:04,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552364344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:04,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:04,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:04,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T02:45:04,414 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:04,414 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:04,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-14T02:45:04,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T02:45:04,416 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:04,416 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:04,417 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:04,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T02:45:04,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:04,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552364553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:04,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:04,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552364553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:04,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:04,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:04,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:04,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552364553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:04,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:04,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:04,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552364553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:04,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:04,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T02:45:04,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:04,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:45:04,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:04,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:04,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:04,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:04,587 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:04,590 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144d49e1b97483458b9215f8eae6535f64_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144d49e1b97483458b9215f8eae6535f64_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:04,591 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/6b891c790e824a46b150a3d5969ad048, store: [table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:45:04,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/6b891c790e824a46b150a3d5969ad048 is 175, key is test_row_0/A:col10/1731552304175/Put/seqid=0 2024-11-14T02:45:04,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742191_1367 (size=39749) 2024-11-14T02:45:04,601 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/6b891c790e824a46b150a3d5969ad048 2024-11-14T02:45:04,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/18a3e74115494d338fff71131847fb50 is 50, key is test_row_0/B:col10/1731552304175/Put/seqid=0 2024-11-14T02:45:04,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742192_1368 (size=12151) 2024-11-14T02:45:04,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T02:45:04,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:04,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:45:04,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:04,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:04,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:04,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:04,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T02:45:04,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:04,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552364864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:04,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:04,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:04,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552364864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:04,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:04,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:04,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552364864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:04,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:04,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T02:45:04,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:04,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:45:04,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:04,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:04,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:04,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:04,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:04,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552364873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:04,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:04,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:05,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/18a3e74115494d338fff71131847fb50 2024-11-14T02:45:05,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/8909c3debf544398969eced9efb8e061 is 50, key is test_row_0/C:col10/1731552304175/Put/seqid=0 2024-11-14T02:45:05,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742193_1369 (size=12151) 2024-11-14T02:45:05,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T02:45:05,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:05,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:45:05,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:05,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:05,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:05,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:05,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T02:45:05,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:05,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1731552365043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:05,051 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_0' in 025639fc1f987c76e816bd52e91790c0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8265 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:45:05,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:05,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:05,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:05,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T02:45:05,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:05,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:45:05,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:05,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:05,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:05,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:05,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T02:45:05,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:05,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:45:05,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:05,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:05,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:05,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552365373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:05,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552365373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:05,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:05,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552365383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:05,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:05,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:05,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:05,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552365393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:05,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/8909c3debf544398969eced9efb8e061 2024-11-14T02:45:05,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/6b891c790e824a46b150a3d5969ad048 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/6b891c790e824a46b150a3d5969ad048 2024-11-14T02:45:05,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/6b891c790e824a46b150a3d5969ad048, entries=200, sequenceid=238, filesize=38.8 K 2024-11-14T02:45:05,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/18a3e74115494d338fff71131847fb50 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/18a3e74115494d338fff71131847fb50 2024-11-14T02:45:05,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/18a3e74115494d338fff71131847fb50, entries=150, sequenceid=238, filesize=11.9 K 2024-11-14T02:45:05,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/8909c3debf544398969eced9efb8e061 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/8909c3debf544398969eced9efb8e061 2024-11-14T02:45:05,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/8909c3debf544398969eced9efb8e061, entries=150, sequenceid=238, filesize=11.9 K 2024-11-14T02:45:05,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 025639fc1f987c76e816bd52e91790c0 in 1262ms, sequenceid=238, compaction requested=true 2024-11-14T02:45:05,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:45:05,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 025639fc1f987c76e816bd52e91790c0:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:45:05,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:05,438 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:05,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 025639fc1f987c76e816bd52e91790c0:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:45:05,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:05,438 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:05,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 025639fc1f987c76e816bd52e91790c0:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:45:05,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:05,439 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:05,439 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:05,439 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 025639fc1f987c76e816bd52e91790c0/B is initiating minor compaction (all files) 2024-11-14T02:45:05,439 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 025639fc1f987c76e816bd52e91790c0/A is initiating minor compaction (all files) 2024-11-14T02:45:05,439 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 025639fc1f987c76e816bd52e91790c0/B in TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:05,439 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 025639fc1f987c76e816bd52e91790c0/A in TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:05,439 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/cfab71f76a3f491d857991e9985f413c, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/40b7e9d05d16406c94522577523e92f2, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/18a3e74115494d338fff71131847fb50] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp, totalSize=36.0 K 2024-11-14T02:45:05,439 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/ebe60dac90404f7587358250cd916d84, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d78dca422675490b9bd2ea68119c1088, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/6b891c790e824a46b150a3d5969ad048] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp, totalSize=100.0 K 2024-11-14T02:45:05,439 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:05,439 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. files: [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/ebe60dac90404f7587358250cd916d84, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d78dca422675490b9bd2ea68119c1088, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/6b891c790e824a46b150a3d5969ad048] 2024-11-14T02:45:05,439 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting cfab71f76a3f491d857991e9985f413c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731552300166 2024-11-14T02:45:05,439 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebe60dac90404f7587358250cd916d84, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731552300166 2024-11-14T02:45:05,440 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 40b7e9d05d16406c94522577523e92f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731552301317 2024-11-14T02:45:05,440 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 18a3e74115494d338fff71131847fb50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731552303530 2024-11-14T02:45:05,440 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting d78dca422675490b9bd2ea68119c1088, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731552301317 2024-11-14T02:45:05,442 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b891c790e824a46b150a3d5969ad048, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731552303530 2024-11-14T02:45:05,447 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 025639fc1f987c76e816bd52e91790c0#B#compaction#308 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:05,448 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/4fc1cee8eec247c8b95e395dfcb1b871 is 50, key is test_row_0/B:col10/1731552304175/Put/seqid=0 2024-11-14T02:45:05,450 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:45:05,453 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111487c6a9f593564026be4b116e2f97eac7_025639fc1f987c76e816bd52e91790c0 store=[table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:45:05,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742194_1370 (size=12697) 2024-11-14T02:45:05,455 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111487c6a9f593564026be4b116e2f97eac7_025639fc1f987c76e816bd52e91790c0, store=[table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:45:05,455 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111487c6a9f593564026be4b116e2f97eac7_025639fc1f987c76e816bd52e91790c0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:45:05,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742195_1371 (size=4469) 2024-11-14T02:45:05,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T02:45:05,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:05,487 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 025639fc1f987c76e816bd52e91790c0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T02:45:05,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cf42ac87835641e6bbcab7f8283b1e25_025639fc1f987c76e816bd52e91790c0 is 50, key is test_row_0/A:col10/1731552304223/Put/seqid=0 2024-11-14T02:45:05,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742196_1372 (size=12304) 2024-11-14T02:45:05,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T02:45:05,858 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/4fc1cee8eec247c8b95e395dfcb1b871 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/4fc1cee8eec247c8b95e395dfcb1b871 2024-11-14T02:45:05,861 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 025639fc1f987c76e816bd52e91790c0#A#compaction#309 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:05,861 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 025639fc1f987c76e816bd52e91790c0/B of 025639fc1f987c76e816bd52e91790c0 into 4fc1cee8eec247c8b95e395dfcb1b871(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:05,862 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:45:05,862 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., storeName=025639fc1f987c76e816bd52e91790c0/B, priority=13, startTime=1731552305438; duration=0sec 2024-11-14T02:45:05,862 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/afb0822413ed485a9a46e647ea25b303 is 175, key is test_row_0/A:col10/1731552304175/Put/seqid=0 2024-11-14T02:45:05,862 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:05,862 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 025639fc1f987c76e816bd52e91790c0:B 2024-11-14T02:45:05,862 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:05,863 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:05,863 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 025639fc1f987c76e816bd52e91790c0/C is initiating minor compaction (all files) 2024-11-14T02:45:05,863 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 025639fc1f987c76e816bd52e91790c0/C in TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:05,863 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/fffe715202fe439cb29f7ab2de5d9b21, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/92c4752f1c364b0caefa5e6e53cc2254, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/8909c3debf544398969eced9efb8e061] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp, totalSize=36.0 K 2024-11-14T02:45:05,864 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting fffe715202fe439cb29f7ab2de5d9b21, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731552300166 2024-11-14T02:45:05,864 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 92c4752f1c364b0caefa5e6e53cc2254, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731552301317 2024-11-14T02:45:05,864 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 8909c3debf544398969eced9efb8e061, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731552303530 2024-11-14T02:45:05,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742197_1373 (size=31651) 2024-11-14T02:45:05,872 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 025639fc1f987c76e816bd52e91790c0#C#compaction#311 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:05,873 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/ae5afe133ed54b1c8d026af0fb6b84fa is 50, key is test_row_0/C:col10/1731552304175/Put/seqid=0 2024-11-14T02:45:05,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742198_1374 (size=12697) 2024-11-14T02:45:05,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:05,901 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cf42ac87835641e6bbcab7f8283b1e25_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cf42ac87835641e6bbcab7f8283b1e25_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:05,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/39a9cbd7f8374ce8b94a8de1c9bbaa61, store: [table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:45:05,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/39a9cbd7f8374ce8b94a8de1c9bbaa61 is 175, key is test_row_0/A:col10/1731552304223/Put/seqid=0 2024-11-14T02:45:05,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742199_1375 (size=31105) 2024-11-14T02:45:06,270 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/afb0822413ed485a9a46e647ea25b303 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/afb0822413ed485a9a46e647ea25b303 2024-11-14T02:45:06,275 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 025639fc1f987c76e816bd52e91790c0/A of 025639fc1f987c76e816bd52e91790c0 into afb0822413ed485a9a46e647ea25b303(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:06,275 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:45:06,275 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., storeName=025639fc1f987c76e816bd52e91790c0/A, priority=13, startTime=1731552305438; duration=0sec 2024-11-14T02:45:06,275 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:06,275 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 025639fc1f987c76e816bd52e91790c0:A 2024-11-14T02:45:06,284 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/ae5afe133ed54b1c8d026af0fb6b84fa as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/ae5afe133ed54b1c8d026af0fb6b84fa 2024-11-14T02:45:06,288 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 025639fc1f987c76e816bd52e91790c0/C of 025639fc1f987c76e816bd52e91790c0 into ae5afe133ed54b1c8d026af0fb6b84fa(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:06,288 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:45:06,288 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., storeName=025639fc1f987c76e816bd52e91790c0/C, priority=13, startTime=1731552305438; duration=0sec 2024-11-14T02:45:06,288 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:06,288 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 025639fc1f987c76e816bd52e91790c0:C 2024-11-14T02:45:06,307 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/39a9cbd7f8374ce8b94a8de1c9bbaa61 2024-11-14T02:45:06,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/45ca967b83704a518f9b8538d0a781a8 is 50, key is test_row_0/B:col10/1731552304223/Put/seqid=0 2024-11-14T02:45:06,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742200_1376 (size=12151) 2024-11-14T02:45:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:06,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. as already flushing 2024-11-14T02:45:06,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552366431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:06,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:06,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:06,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:06,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552366432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:06,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:06,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:06,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:06,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552366433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:06,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:06,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552366437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:06,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:06,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:06,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:06,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:06,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:06,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552366543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:06,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:06,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552366543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:06,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:06,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:06,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:06,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:06,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552366543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:06,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:06,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:06,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:06,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:06,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:06,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:06,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552366543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:06,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:06,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:06,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T02:45:06,720 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/45ca967b83704a518f9b8538d0a781a8 2024-11-14T02:45:06,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/e8b8176405584458aed433a44838f66a is 50, key is test_row_0/C:col10/1731552304223/Put/seqid=0 2024-11-14T02:45:06,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742201_1377 (size=12151) 2024-11-14T02:45:06,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:06,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552366753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:06,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:06,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:06,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:06,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552366753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:06,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:06,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:06,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:06,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:06,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552366753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:06,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:06,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:06,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:06,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:06,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:06,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552366753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:06,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:06,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:06,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:07,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552367063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552367063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:07,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552367063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:07,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552367063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:07,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:07,129 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/e8b8176405584458aed433a44838f66a 2024-11-14T02:45:07,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/39a9cbd7f8374ce8b94a8de1c9bbaa61 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/39a9cbd7f8374ce8b94a8de1c9bbaa61 2024-11-14T02:45:07,135 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/39a9cbd7f8374ce8b94a8de1c9bbaa61, entries=150, sequenceid=249, filesize=30.4 K 2024-11-14T02:45:07,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/45ca967b83704a518f9b8538d0a781a8 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/45ca967b83704a518f9b8538d0a781a8 2024-11-14T02:45:07,139 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/45ca967b83704a518f9b8538d0a781a8, entries=150, sequenceid=249, filesize=11.9 K 2024-11-14T02:45:07,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/e8b8176405584458aed433a44838f66a as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/e8b8176405584458aed433a44838f66a 2024-11-14T02:45:07,143 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/e8b8176405584458aed433a44838f66a, entries=150, sequenceid=249, filesize=11.9 K 2024-11-14T02:45:07,144 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 025639fc1f987c76e816bd52e91790c0 in 1657ms, sequenceid=249, compaction requested=false 2024-11-14T02:45:07,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:45:07,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:07,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-14T02:45:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-14T02:45:07,147 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-14T02:45:07,147 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7290 sec 2024-11-14T02:45:07,149 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.7340 sec 2024-11-14T02:45:07,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:07,577 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:45:07,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 025639fc1f987c76e816bd52e91790c0 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-14T02:45:07,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:07,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552367583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:07,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114006617d889a04c8193280215dc8f84ac_025639fc1f987c76e816bd52e91790c0 is 50, key is test_row_0/A:col10/1731552306431/Put/seqid=0 2024-11-14T02:45:07,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:07,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552367588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:07,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:07,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552367588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:07,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:07,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:07,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552367589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:07,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:07,608 INFO [Thread-1426 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1426. 2024-11-14T02:45:07,608 DEBUG [Thread-1426 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:07,608 DEBUG [Thread-1426 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:07,608 DEBUG [Thread-1426 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:07,608 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:07,611 INFO [Thread-1420 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1420. 2024-11-14T02:45:07,611 DEBUG [Thread-1420 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:07,611 DEBUG [Thread-1420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:07,611 DEBUG [Thread-1420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:07,611 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:07,612 INFO [Thread-1428 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1428. 2024-11-14T02:45:07,612 DEBUG [Thread-1428 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:07,612 DEBUG [Thread-1428 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:07,612 DEBUG [Thread-1428 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:07,612 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:07,616 INFO [Thread-1422 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1422. 2024-11-14T02:45:07,616 DEBUG [Thread-1422 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:07,616 DEBUG [Thread-1422 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:07,616 DEBUG [Thread-1422 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:07,616 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:07,619 INFO [Thread-1424 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1424. 2024-11-14T02:45:07,619 DEBUG [Thread-1424 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:07,619 DEBUG [Thread-1424 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:07,619 DEBUG [Thread-1424 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:07,619 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:07,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742202_1378 (size=17534) 2024-11-14T02:45:07,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:07,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:07,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552367694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:07,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552367694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:07,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:07,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:07,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:07,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552367704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:07,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:07,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552367704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:07,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:07,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:07,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:07,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:07,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552367905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:07,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552367905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:07,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:07,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:07,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:07,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552367914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:07,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:07,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552367915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:07,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:07,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:07,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:08,022 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:08,025 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114006617d889a04c8193280215dc8f84ac_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114006617d889a04c8193280215dc8f84ac_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:08,026 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/d1ef97018b374b669674baa9c74ebcaa, store: [table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:45:08,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/d1ef97018b374b669674baa9c74ebcaa is 175, key is test_row_0/A:col10/1731552306431/Put/seqid=0 2024-11-14T02:45:08,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742203_1379 (size=48639) 2024-11-14T02:45:08,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:08,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:08,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552368214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:08,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552368214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:08,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:08,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552368224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:08,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:08,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552368224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:08,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:08,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:08,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:08,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:08,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:08,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:08,431 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/d1ef97018b374b669674baa9c74ebcaa 2024-11-14T02:45:08,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/e2d390479a214bc89cc44a4e2db8c6ef is 50, key is test_row_0/B:col10/1731552306431/Put/seqid=0 2024-11-14T02:45:08,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742204_1380 (size=12301) 2024-11-14T02:45:08,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T02:45:08,585 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:08,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:08,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50968 deadline: 1731552368725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51040 deadline: 1731552368725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:08,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:08,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50986 deadline: 1731552368734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:08,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:08,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51034 deadline: 1731552368735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:08,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/e2d390479a214bc89cc44a4e2db8c6ef 2024-11-14T02:45:08,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/01dd39ea1801404c86c8ba739d74f054 is 50, key is test_row_0/C:col10/1731552306431/Put/seqid=0 2024-11-14T02:45:08,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742205_1381 (size=12301) 2024-11-14T02:45:09,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/01dd39ea1801404c86c8ba739d74f054 2024-11-14T02:45:09,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/d1ef97018b374b669674baa9c74ebcaa as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d1ef97018b374b669674baa9c74ebcaa 2024-11-14T02:45:09,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d1ef97018b374b669674baa9c74ebcaa, entries=250, sequenceid=279, filesize=47.5 K 2024-11-14T02:45:09,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/e2d390479a214bc89cc44a4e2db8c6ef as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/e2d390479a214bc89cc44a4e2db8c6ef 2024-11-14T02:45:09,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/e2d390479a214bc89cc44a4e2db8c6ef, entries=150, sequenceid=279, filesize=12.0 K 2024-11-14T02:45:09,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/01dd39ea1801404c86c8ba739d74f054 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/01dd39ea1801404c86c8ba739d74f054 2024-11-14T02:45:09,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/01dd39ea1801404c86c8ba739d74f054, entries=150, sequenceid=279, filesize=12.0 K 2024-11-14T02:45:09,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 025639fc1f987c76e816bd52e91790c0 in 1714ms, sequenceid=279, compaction requested=true 2024-11-14T02:45:09,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:45:09,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 025639fc1f987c76e816bd52e91790c0:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:45:09,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:09,292 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:09,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 025639fc1f987c76e816bd52e91790c0:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:45:09,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:09,292 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:09,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 025639fc1f987c76e816bd52e91790c0:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:45:09,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:09,293 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111395 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:09,293 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:09,293 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 025639fc1f987c76e816bd52e91790c0/B is initiating minor compaction (all files) 2024-11-14T02:45:09,293 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 025639fc1f987c76e816bd52e91790c0/A is initiating minor compaction (all files) 2024-11-14T02:45:09,293 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 025639fc1f987c76e816bd52e91790c0/B in TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:09,293 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 025639fc1f987c76e816bd52e91790c0/A in TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:09,293 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/afb0822413ed485a9a46e647ea25b303, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/39a9cbd7f8374ce8b94a8de1c9bbaa61, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d1ef97018b374b669674baa9c74ebcaa] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp, totalSize=108.8 K 2024-11-14T02:45:09,293 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/4fc1cee8eec247c8b95e395dfcb1b871, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/45ca967b83704a518f9b8538d0a781a8, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/e2d390479a214bc89cc44a4e2db8c6ef] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp, totalSize=36.3 K 2024-11-14T02:45:09,293 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:09,293 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. files: [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/afb0822413ed485a9a46e647ea25b303, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/39a9cbd7f8374ce8b94a8de1c9bbaa61, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d1ef97018b374b669674baa9c74ebcaa] 2024-11-14T02:45:09,293 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fc1cee8eec247c8b95e395dfcb1b871, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731552303530 2024-11-14T02:45:09,293 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting afb0822413ed485a9a46e647ea25b303, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731552303530 2024-11-14T02:45:09,294 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 45ca967b83704a518f9b8538d0a781a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731552304221 2024-11-14T02:45:09,294 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39a9cbd7f8374ce8b94a8de1c9bbaa61, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731552304221 2024-11-14T02:45:09,294 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting e2d390479a214bc89cc44a4e2db8c6ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731552306431 2024-11-14T02:45:09,294 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1ef97018b374b669674baa9c74ebcaa, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731552306404 2024-11-14T02:45:09,301 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:45:09,302 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 025639fc1f987c76e816bd52e91790c0#B#compaction#317 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:09,302 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/13457d3c4a7f46c395999d4700b2824f is 50, key is test_row_0/B:col10/1731552306431/Put/seqid=0 2024-11-14T02:45:09,303 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114ba54481efc32410c96f36d9f19d92d7c_025639fc1f987c76e816bd52e91790c0 store=[table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:45:09,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742206_1382 (size=12949) 2024-11-14T02:45:09,311 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/13457d3c4a7f46c395999d4700b2824f as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/13457d3c4a7f46c395999d4700b2824f 2024-11-14T02:45:09,315 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 025639fc1f987c76e816bd52e91790c0/B of 025639fc1f987c76e816bd52e91790c0 into 13457d3c4a7f46c395999d4700b2824f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:09,315 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:45:09,315 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., storeName=025639fc1f987c76e816bd52e91790c0/B, priority=13, startTime=1731552309292; duration=0sec 2024-11-14T02:45:09,315 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:09,315 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 025639fc1f987c76e816bd52e91790c0:B 2024-11-14T02:45:09,315 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:09,316 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:09,316 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 025639fc1f987c76e816bd52e91790c0/C is initiating minor compaction (all files) 2024-11-14T02:45:09,316 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 025639fc1f987c76e816bd52e91790c0/C in TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:09,316 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/ae5afe133ed54b1c8d026af0fb6b84fa, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/e8b8176405584458aed433a44838f66a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/01dd39ea1801404c86c8ba739d74f054] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp, totalSize=36.3 K 2024-11-14T02:45:09,317 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting ae5afe133ed54b1c8d026af0fb6b84fa, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731552303530 2024-11-14T02:45:09,317 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting e8b8176405584458aed433a44838f66a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731552304221 2024-11-14T02:45:09,317 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 01dd39ea1801404c86c8ba739d74f054, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731552306431 2024-11-14T02:45:09,333 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 025639fc1f987c76e816bd52e91790c0#C#compaction#319 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:09,333 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/d790e2cc603e42fe85eb24aa11c8be9f is 50, key is test_row_0/C:col10/1731552306431/Put/seqid=0 2024-11-14T02:45:09,334 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114ba54481efc32410c96f36d9f19d92d7c_025639fc1f987c76e816bd52e91790c0, store=[table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:45:09,334 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ba54481efc32410c96f36d9f19d92d7c_025639fc1f987c76e816bd52e91790c0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:45:09,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742207_1383 (size=12949) 2024-11-14T02:45:09,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742208_1384 (size=4469) 2024-11-14T02:45:09,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:09,737 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:45:09,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 025639fc1f987c76e816bd52e91790c0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T02:45:09,738 INFO [Thread-1409 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1409. 2024-11-14T02:45:09,738 INFO [Thread-1417 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1417. 2024-11-14T02:45:09,738 DEBUG [Thread-1409 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:09,738 DEBUG [Thread-1417 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:09,738 DEBUG [Thread-1409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:09,738 DEBUG [Thread-1417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:09,738 DEBUG [Thread-1409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:09,738 DEBUG [Thread-1417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:09,738 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:09,738 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:09,742 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/d790e2cc603e42fe85eb24aa11c8be9f as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/d790e2cc603e42fe85eb24aa11c8be9f 2024-11-14T02:45:09,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111483ba32954e454272bfbfd869815ee5ee_025639fc1f987c76e816bd52e91790c0 is 50, key is test_row_0/A:col10/1731552309734/Put/seqid=0 2024-11-14T02:45:09,745 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 025639fc1f987c76e816bd52e91790c0#A#compaction#318 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:09,746 INFO [Thread-1415 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1415. 2024-11-14T02:45:09,746 DEBUG [Thread-1415 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:09,746 DEBUG [Thread-1415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:09,746 DEBUG [Thread-1415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:09,746 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/67403d1439804aa7b24218087dc2768e is 175, key is test_row_0/A:col10/1731552306431/Put/seqid=0 2024-11-14T02:45:09,746 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:09,748 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 025639fc1f987c76e816bd52e91790c0/C of 025639fc1f987c76e816bd52e91790c0 into d790e2cc603e42fe85eb24aa11c8be9f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:09,748 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:45:09,748 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., storeName=025639fc1f987c76e816bd52e91790c0/C, priority=13, startTime=1731552309292; duration=0sec 2024-11-14T02:45:09,748 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:09,749 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 025639fc1f987c76e816bd52e91790c0:C 2024-11-14T02:45:09,750 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T02:45:09,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742210_1386 (size=31903) 2024-11-14T02:45:09,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742209_1385 (size=12454) 2024-11-14T02:45:09,755 INFO [Thread-1413 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1413. 2024-11-14T02:45:09,755 DEBUG [Thread-1413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:09,755 DEBUG [Thread-1413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:09,755 DEBUG [Thread-1413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:09,755 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:10,156 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:10,163 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111483ba32954e454272bfbfd869815ee5ee_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111483ba32954e454272bfbfd869815ee5ee_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:10,165 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/67403d1439804aa7b24218087dc2768e as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/67403d1439804aa7b24218087dc2768e 2024-11-14T02:45:10,165 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/f90d50bcdaf44acf96d054f2176c6e84, store: [table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:45:10,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/f90d50bcdaf44acf96d054f2176c6e84 is 175, key is test_row_0/A:col10/1731552309734/Put/seqid=0 2024-11-14T02:45:10,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742211_1387 (size=31255) 2024-11-14T02:45:10,170 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 025639fc1f987c76e816bd52e91790c0/A of 025639fc1f987c76e816bd52e91790c0 into 67403d1439804aa7b24218087dc2768e(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:10,171 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:45:10,171 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., storeName=025639fc1f987c76e816bd52e91790c0/A, priority=13, startTime=1731552309292; duration=0sec 2024-11-14T02:45:10,171 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:10,171 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 025639fc1f987c76e816bd52e91790c0:A 2024-11-14T02:45:10,571 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/f90d50bcdaf44acf96d054f2176c6e84 2024-11-14T02:45:10,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/42afbd39213147c3bcc81441e179c6ec is 50, key is test_row_0/B:col10/1731552309734/Put/seqid=0 2024-11-14T02:45:10,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742212_1388 (size=12301) 2024-11-14T02:45:10,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/42afbd39213147c3bcc81441e179c6ec 2024-11-14T02:45:11,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/31c5cc53be804d01beccc9fb274f811c is 50, key is test_row_0/C:col10/1731552309734/Put/seqid=0 2024-11-14T02:45:11,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742213_1389 (size=12301) 2024-11-14T02:45:11,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/31c5cc53be804d01beccc9fb274f811c 2024-11-14T02:45:11,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/f90d50bcdaf44acf96d054f2176c6e84 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/f90d50bcdaf44acf96d054f2176c6e84 2024-11-14T02:45:11,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/f90d50bcdaf44acf96d054f2176c6e84, entries=150, sequenceid=291, filesize=30.5 K 2024-11-14T02:45:11,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/42afbd39213147c3bcc81441e179c6ec as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/42afbd39213147c3bcc81441e179c6ec 2024-11-14T02:45:11,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/42afbd39213147c3bcc81441e179c6ec, entries=150, sequenceid=291, filesize=12.0 K 2024-11-14T02:45:11,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/31c5cc53be804d01beccc9fb274f811c as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/31c5cc53be804d01beccc9fb274f811c 2024-11-14T02:45:11,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/31c5cc53be804d01beccc9fb274f811c, entries=150, sequenceid=291, filesize=12.0 K 2024-11-14T02:45:11,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=13.42 KB/13740 for 025639fc1f987c76e816bd52e91790c0 in 1696ms, sequenceid=291, compaction requested=false 2024-11-14T02:45:11,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 025639fc1f987c76e816bd52e91790c0: 2024-11-14T02:45:14,172 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d8f61c14e4ed4b2c8aa30e46b2cbb763, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/2fae77053b9847478b445a21a4dcb9a3, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/7c6554028edf49369a25150271d2cba9, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/2a064daf117a49e284fd0107acf6770c, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/8304a2838e9b46f58fb61d68b541bbd7, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/4daeeed6a61f40d48f03d9228be5e26a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/0e9b42213e0040ab83a30f9b5c044c7f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/4b0149e551534573bb20b9ac5b0ef911, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/cdd69051d77d40bea077532de86335f4, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/5544312e1ee1418f8d0246a089b09e7b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/503e708177a84275b0d6125fe01463a4, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/cfe2722f60f74560b77649a05ad0ea55, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d1d341a514bc45058a0676665d38a2fa, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/ebe60dac90404f7587358250cd916d84, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d78dca422675490b9bd2ea68119c1088, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/6b891c790e824a46b150a3d5969ad048, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/afb0822413ed485a9a46e647ea25b303, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/39a9cbd7f8374ce8b94a8de1c9bbaa61, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d1ef97018b374b669674baa9c74ebcaa] to archive 2024-11-14T02:45:14,175 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:45:14,179 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d8f61c14e4ed4b2c8aa30e46b2cbb763 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d8f61c14e4ed4b2c8aa30e46b2cbb763 2024-11-14T02:45:14,180 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/2fae77053b9847478b445a21a4dcb9a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/2fae77053b9847478b445a21a4dcb9a3 2024-11-14T02:45:14,181 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/7c6554028edf49369a25150271d2cba9 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/7c6554028edf49369a25150271d2cba9 2024-11-14T02:45:14,183 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/2a064daf117a49e284fd0107acf6770c to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/2a064daf117a49e284fd0107acf6770c 2024-11-14T02:45:14,184 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/8304a2838e9b46f58fb61d68b541bbd7 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/8304a2838e9b46f58fb61d68b541bbd7 2024-11-14T02:45:14,185 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/4daeeed6a61f40d48f03d9228be5e26a to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/4daeeed6a61f40d48f03d9228be5e26a 2024-11-14T02:45:14,186 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/0e9b42213e0040ab83a30f9b5c044c7f to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/0e9b42213e0040ab83a30f9b5c044c7f 2024-11-14T02:45:14,187 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/4b0149e551534573bb20b9ac5b0ef911 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/4b0149e551534573bb20b9ac5b0ef911 2024-11-14T02:45:14,188 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/cdd69051d77d40bea077532de86335f4 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/cdd69051d77d40bea077532de86335f4 2024-11-14T02:45:14,189 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/5544312e1ee1418f8d0246a089b09e7b to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/5544312e1ee1418f8d0246a089b09e7b 2024-11-14T02:45:14,190 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/503e708177a84275b0d6125fe01463a4 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/503e708177a84275b0d6125fe01463a4 2024-11-14T02:45:14,191 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/cfe2722f60f74560b77649a05ad0ea55 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/cfe2722f60f74560b77649a05ad0ea55 2024-11-14T02:45:14,191 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d1d341a514bc45058a0676665d38a2fa to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d1d341a514bc45058a0676665d38a2fa 2024-11-14T02:45:14,192 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/ebe60dac90404f7587358250cd916d84 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/ebe60dac90404f7587358250cd916d84 2024-11-14T02:45:14,193 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d78dca422675490b9bd2ea68119c1088 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d78dca422675490b9bd2ea68119c1088 2024-11-14T02:45:14,194 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/6b891c790e824a46b150a3d5969ad048 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/6b891c790e824a46b150a3d5969ad048 2024-11-14T02:45:14,195 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/afb0822413ed485a9a46e647ea25b303 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/afb0822413ed485a9a46e647ea25b303 2024-11-14T02:45:14,196 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/39a9cbd7f8374ce8b94a8de1c9bbaa61 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/39a9cbd7f8374ce8b94a8de1c9bbaa61 2024-11-14T02:45:14,197 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d1ef97018b374b669674baa9c74ebcaa to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/d1ef97018b374b669674baa9c74ebcaa 2024-11-14T02:45:14,200 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/10f55b3e62764b8cafda603ccc41e447, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/0a6e2743d51849e4b1cb457a8fc8e613, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/e6064d0e2d394cf08c883deeb49b5317, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/e0841ab766254b9186b776846b87d760, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/ece360bf8f1343ec9c119d22994051b8, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/8d2a0109c7354787b2c00050ee93556d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/bc30869f7f584e5cad1deff7bfe402fa, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/3576ee5713e2465a93d119326bebc21b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/b5dc534315434f8ea75ce2a25f740b71, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/c510a687d228429ab916a5716030f26e, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/0e97659e64f7455ebb8299db7837a274, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/4f1b4683d1f7420ba5aa3550db4158ec, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/cfab71f76a3f491d857991e9985f413c, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/d22ee03f86ec485e972f08ef41485755, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/40b7e9d05d16406c94522577523e92f2, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/4fc1cee8eec247c8b95e395dfcb1b871, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/18a3e74115494d338fff71131847fb50, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/45ca967b83704a518f9b8538d0a781a8, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/e2d390479a214bc89cc44a4e2db8c6ef] to archive 2024-11-14T02:45:14,200 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:45:14,202 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/10f55b3e62764b8cafda603ccc41e447 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/10f55b3e62764b8cafda603ccc41e447 2024-11-14T02:45:14,202 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/0a6e2743d51849e4b1cb457a8fc8e613 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/0a6e2743d51849e4b1cb457a8fc8e613 2024-11-14T02:45:14,203 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/e6064d0e2d394cf08c883deeb49b5317 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/e6064d0e2d394cf08c883deeb49b5317 2024-11-14T02:45:14,204 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/e0841ab766254b9186b776846b87d760 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/e0841ab766254b9186b776846b87d760 2024-11-14T02:45:14,204 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/ece360bf8f1343ec9c119d22994051b8 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/ece360bf8f1343ec9c119d22994051b8 2024-11-14T02:45:14,205 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/8d2a0109c7354787b2c00050ee93556d to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/8d2a0109c7354787b2c00050ee93556d 2024-11-14T02:45:14,206 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/bc30869f7f584e5cad1deff7bfe402fa to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/bc30869f7f584e5cad1deff7bfe402fa 2024-11-14T02:45:14,207 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/3576ee5713e2465a93d119326bebc21b to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/3576ee5713e2465a93d119326bebc21b 2024-11-14T02:45:14,208 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/b5dc534315434f8ea75ce2a25f740b71 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/b5dc534315434f8ea75ce2a25f740b71 2024-11-14T02:45:14,209 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/c510a687d228429ab916a5716030f26e to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/c510a687d228429ab916a5716030f26e 2024-11-14T02:45:14,210 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/0e97659e64f7455ebb8299db7837a274 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/0e97659e64f7455ebb8299db7837a274 2024-11-14T02:45:14,210 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/4f1b4683d1f7420ba5aa3550db4158ec to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/4f1b4683d1f7420ba5aa3550db4158ec 2024-11-14T02:45:14,211 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/cfab71f76a3f491d857991e9985f413c to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/cfab71f76a3f491d857991e9985f413c 2024-11-14T02:45:14,212 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/d22ee03f86ec485e972f08ef41485755 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/d22ee03f86ec485e972f08ef41485755 2024-11-14T02:45:14,212 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/40b7e9d05d16406c94522577523e92f2 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/40b7e9d05d16406c94522577523e92f2 2024-11-14T02:45:14,213 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/4fc1cee8eec247c8b95e395dfcb1b871 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/4fc1cee8eec247c8b95e395dfcb1b871 2024-11-14T02:45:14,214 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/18a3e74115494d338fff71131847fb50 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/18a3e74115494d338fff71131847fb50 2024-11-14T02:45:14,214 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/45ca967b83704a518f9b8538d0a781a8 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/45ca967b83704a518f9b8538d0a781a8 2024-11-14T02:45:14,215 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/e2d390479a214bc89cc44a4e2db8c6ef to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/e2d390479a214bc89cc44a4e2db8c6ef 2024-11-14T02:45:14,217 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/4c2837c1705248f2b185603d75e6dafd, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/dd69135deab94cb996c1ee399f3af4c3, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/8a6b3e767f2d40dc88a856971c266f27, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/155b8be1dc3244eab456afd8e5d61c91, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/721cc64a63bb439db31802f3aacce79e, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/a74c88847781457ca02dd3f3d2f4ca68, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/3367fd6782c3484d93f6ab280c41703b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/d9f7916095b940518bc869ed1eb75787, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/8f392c164c934580bbd99442f961be67, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/5c246bb8ca424d48b37f4136ee50da97, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/1ed5c1e865684bc0b85aa807b5c45168, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/43fc37146cd14d9da86bfdca4636a314, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/fffe715202fe439cb29f7ab2de5d9b21, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/feaeb18f657e463e9aaa14b6fd036262, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/92c4752f1c364b0caefa5e6e53cc2254, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/ae5afe133ed54b1c8d026af0fb6b84fa, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/8909c3debf544398969eced9efb8e061, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/e8b8176405584458aed433a44838f66a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/01dd39ea1801404c86c8ba739d74f054] to archive 2024-11-14T02:45:14,218 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:45:14,219 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/4c2837c1705248f2b185603d75e6dafd to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/4c2837c1705248f2b185603d75e6dafd 2024-11-14T02:45:14,220 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/dd69135deab94cb996c1ee399f3af4c3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/dd69135deab94cb996c1ee399f3af4c3 2024-11-14T02:45:14,221 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/8a6b3e767f2d40dc88a856971c266f27 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/8a6b3e767f2d40dc88a856971c266f27 2024-11-14T02:45:14,222 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/155b8be1dc3244eab456afd8e5d61c91 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/155b8be1dc3244eab456afd8e5d61c91 2024-11-14T02:45:14,223 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/721cc64a63bb439db31802f3aacce79e to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/721cc64a63bb439db31802f3aacce79e 2024-11-14T02:45:14,223 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/a74c88847781457ca02dd3f3d2f4ca68 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/a74c88847781457ca02dd3f3d2f4ca68 2024-11-14T02:45:14,224 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/3367fd6782c3484d93f6ab280c41703b to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/3367fd6782c3484d93f6ab280c41703b 2024-11-14T02:45:14,225 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/d9f7916095b940518bc869ed1eb75787 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/d9f7916095b940518bc869ed1eb75787 2024-11-14T02:45:14,225 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/8f392c164c934580bbd99442f961be67 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/8f392c164c934580bbd99442f961be67 2024-11-14T02:45:14,226 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/5c246bb8ca424d48b37f4136ee50da97 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/5c246bb8ca424d48b37f4136ee50da97 2024-11-14T02:45:14,227 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/1ed5c1e865684bc0b85aa807b5c45168 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/1ed5c1e865684bc0b85aa807b5c45168 2024-11-14T02:45:14,227 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/43fc37146cd14d9da86bfdca4636a314 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/43fc37146cd14d9da86bfdca4636a314 2024-11-14T02:45:14,228 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/fffe715202fe439cb29f7ab2de5d9b21 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/fffe715202fe439cb29f7ab2de5d9b21 2024-11-14T02:45:14,229 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/feaeb18f657e463e9aaa14b6fd036262 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/feaeb18f657e463e9aaa14b6fd036262 2024-11-14T02:45:14,229 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/92c4752f1c364b0caefa5e6e53cc2254 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/92c4752f1c364b0caefa5e6e53cc2254 2024-11-14T02:45:14,230 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/ae5afe133ed54b1c8d026af0fb6b84fa to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/ae5afe133ed54b1c8d026af0fb6b84fa 2024-11-14T02:45:14,230 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/8909c3debf544398969eced9efb8e061 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/8909c3debf544398969eced9efb8e061 2024-11-14T02:45:14,231 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/e8b8176405584458aed433a44838f66a to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/e8b8176405584458aed433a44838f66a 2024-11-14T02:45:14,232 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/83c5a5c119d5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/01dd39ea1801404c86c8ba739d74f054 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/01dd39ea1801404c86c8ba739d74f054 2024-11-14T02:45:15,128 INFO [Thread-1411 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1411. 2024-11-14T02:45:15,129 DEBUG [Thread-1411 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:15,129 DEBUG [Thread-1411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:15,129 DEBUG [Thread-1411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:15,129 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:15,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T02:45:15,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-14T02:45:15,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-14T02:45:15,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-14T02:45:15,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-14T02:45:15,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-14T02:45:15,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T02:45:15,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T02:45:15,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1632 2024-11-14T02:45:15,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4895 rows 2024-11-14T02:45:15,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1625 2024-11-14T02:45:15,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4874 rows 2024-11-14T02:45:15,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1636 2024-11-14T02:45:15,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4908 rows 2024-11-14T02:45:15,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1630 2024-11-14T02:45:15,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4889 rows 2024-11-14T02:45:15,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1622 2024-11-14T02:45:15,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4865 rows 2024-11-14T02:45:15,131 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T02:45:15,131 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T02:45:15,131 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:15,131 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:15,132 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:15,133 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T02:45:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:15,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T02:45:15,138 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552315138"}]},"ts":"1731552315138"} 2024-11-14T02:45:15,140 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T02:45:15,140 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T02:45:15,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T02:45:15,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=025639fc1f987c76e816bd52e91790c0, UNASSIGN}] 2024-11-14T02:45:15,142 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=025639fc1f987c76e816bd52e91790c0, UNASSIGN 2024-11-14T02:45:15,143 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=025639fc1f987c76e816bd52e91790c0, regionState=CLOSING, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:15,144 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=025639fc1f987c76e816bd52e91790c0, UNASSIGN because future has completed 2024-11-14T02:45:15,144 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T02:45:15,144 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259}] 2024-11-14T02:45:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T02:45:15,298 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(122): Close 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:15,298 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T02:45:15,298 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1722): Closing 025639fc1f987c76e816bd52e91790c0, disabling compactions & flushes 2024-11-14T02:45:15,298 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:15,299 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:15,299 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. after waiting 0 ms 2024-11-14T02:45:15,299 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:15,299 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(2902): Flushing 025639fc1f987c76e816bd52e91790c0 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-14T02:45:15,312 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144fd1eb2998344e6f97ad2cc7955ffd4d_025639fc1f987c76e816bd52e91790c0 is 50, key is test_row_0/A:col10/1731552315125/Put/seqid=0 2024-11-14T02:45:15,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742214_1390 (size=9914) 2024-11-14T02:45:15,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T02:45:15,717 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:15,728 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144fd1eb2998344e6f97ad2cc7955ffd4d_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144fd1eb2998344e6f97ad2cc7955ffd4d_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:15,730 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/4981a62933504e4d8fbe38a156a3e572, store: [table=TestAcidGuarantees family=A region=025639fc1f987c76e816bd52e91790c0] 2024-11-14T02:45:15,730 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/4981a62933504e4d8fbe38a156a3e572 is 175, key is test_row_0/A:col10/1731552315125/Put/seqid=0 2024-11-14T02:45:15,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742215_1391 (size=22561) 2024-11-14T02:45:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T02:45:16,136 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=299, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/4981a62933504e4d8fbe38a156a3e572 2024-11-14T02:45:16,150 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/b53c021b776c4bf09cee244ede9b3601 is 50, key is test_row_0/B:col10/1731552315125/Put/seqid=0 2024-11-14T02:45:16,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742216_1392 (size=9857) 2024-11-14T02:45:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T02:45:16,555 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/b53c021b776c4bf09cee244ede9b3601 2024-11-14T02:45:16,568 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/f5502f65a3b4473ebd5cbab6a9ebceaa is 50, key is test_row_0/C:col10/1731552315125/Put/seqid=0 2024-11-14T02:45:16,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742217_1393 (size=9857) 2024-11-14T02:45:16,975 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/f5502f65a3b4473ebd5cbab6a9ebceaa 2024-11-14T02:45:16,984 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/A/4981a62933504e4d8fbe38a156a3e572 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/4981a62933504e4d8fbe38a156a3e572 2024-11-14T02:45:16,990 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/4981a62933504e4d8fbe38a156a3e572, entries=100, sequenceid=299, filesize=22.0 K 2024-11-14T02:45:16,991 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/B/b53c021b776c4bf09cee244ede9b3601 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/b53c021b776c4bf09cee244ede9b3601 2024-11-14T02:45:16,995 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/b53c021b776c4bf09cee244ede9b3601, entries=100, sequenceid=299, filesize=9.6 K 2024-11-14T02:45:16,995 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/.tmp/C/f5502f65a3b4473ebd5cbab6a9ebceaa as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/f5502f65a3b4473ebd5cbab6a9ebceaa 2024-11-14T02:45:16,999 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/f5502f65a3b4473ebd5cbab6a9ebceaa, entries=100, sequenceid=299, filesize=9.6 K 2024-11-14T02:45:17,000 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 025639fc1f987c76e816bd52e91790c0 in 1701ms, sequenceid=299, compaction requested=true 2024-11-14T02:45:17,005 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/recovered.edits/302.seqid, newMaxSeqId=302, maxSeqId=4 2024-11-14T02:45:17,005 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. 2024-11-14T02:45:17,005 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1676): Region close journal for 025639fc1f987c76e816bd52e91790c0: Waiting for close lock at 1731552315298Running coprocessor pre-close hooks at 1731552315298Disabling compacts and flushes for region at 1731552315298Disabling writes for close at 1731552315299 (+1 ms)Obtaining lock to block concurrent updates at 1731552315299Preparing flush snapshotting stores in 025639fc1f987c76e816bd52e91790c0 at 1731552315299Finished memstore snapshotting TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731552315300 (+1 ms)Flushing stores of TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0. at 1731552315301 (+1 ms)Flushing 025639fc1f987c76e816bd52e91790c0/A: creating writer at 1731552315302 (+1 ms)Flushing mob file 025639fc1f987c76e816bd52e91790c0/A: appending metadata at 1731552315312 (+10 ms)Flushing mob file 025639fc1f987c76e816bd52e91790c0/A: closing flushed file at 1731552315312Flushing 025639fc1f987c76e816bd52e91790c0/A: appending metadata at 1731552315730 (+418 ms)Flushing 025639fc1f987c76e816bd52e91790c0/A: closing flushed file at 1731552315730Flushing 025639fc1f987c76e816bd52e91790c0/B: creating writer at 1731552316147 (+417 ms)Flushing 025639fc1f987c76e816bd52e91790c0/B: appending metadata at 1731552316150 (+3 ms)Flushing 025639fc1f987c76e816bd52e91790c0/B: closing flushed file at 1731552316150Flushing 025639fc1f987c76e816bd52e91790c0/C: creating writer at 1731552316563 (+413 ms)Flushing 025639fc1f987c76e816bd52e91790c0/C: appending metadata at 1731552316567 (+4 ms)Flushing 025639fc1f987c76e816bd52e91790c0/C: closing flushed file at 1731552316568 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7571e074: reopening flushed file at 1731552316983 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67f1db89: reopening flushed file at 1731552316990 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46c47429: reopening flushed file at 1731552316995 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 025639fc1f987c76e816bd52e91790c0 in 1701ms, sequenceid=299, compaction requested=true at 1731552317000 (+5 ms)Writing region close event to WAL at 1731552317002 (+2 ms)Running coprocessor post-close hooks at 1731552317005 (+3 ms)Closed at 1731552317005 2024-11-14T02:45:17,007 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(157): Closed 025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:17,008 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=025639fc1f987c76e816bd52e91790c0, regionState=CLOSED 2024-11-14T02:45:17,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 because future has completed 2024-11-14T02:45:17,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-14T02:45:17,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseRegionProcedure 025639fc1f987c76e816bd52e91790c0, server=83c5a5c119d5,41455,1731552191259 in 1.8660 sec 2024-11-14T02:45:17,012 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-11-14T02:45:17,012 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=025639fc1f987c76e816bd52e91790c0, UNASSIGN in 1.8690 sec 2024-11-14T02:45:17,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-14T02:45:17,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8730 sec 2024-11-14T02:45:17,015 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552317015"}]},"ts":"1731552317015"} 2024-11-14T02:45:17,016 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T02:45:17,016 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T02:45:17,018 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8830 sec 2024-11-14T02:45:17,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T02:45:17,284 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:17,285 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T02:45:17,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:17,289 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:17,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T02:45:17,291 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=113, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:17,294 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:17,299 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A, FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B, FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C, FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/recovered.edits] 2024-11-14T02:45:17,303 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/4981a62933504e4d8fbe38a156a3e572 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/4981a62933504e4d8fbe38a156a3e572 2024-11-14T02:45:17,305 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/67403d1439804aa7b24218087dc2768e to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/67403d1439804aa7b24218087dc2768e 2024-11-14T02:45:17,307 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/f90d50bcdaf44acf96d054f2176c6e84 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/A/f90d50bcdaf44acf96d054f2176c6e84 2024-11-14T02:45:17,310 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/13457d3c4a7f46c395999d4700b2824f to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/13457d3c4a7f46c395999d4700b2824f 2024-11-14T02:45:17,311 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/42afbd39213147c3bcc81441e179c6ec to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/42afbd39213147c3bcc81441e179c6ec 2024-11-14T02:45:17,313 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/b53c021b776c4bf09cee244ede9b3601 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/B/b53c021b776c4bf09cee244ede9b3601 2024-11-14T02:45:17,316 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/31c5cc53be804d01beccc9fb274f811c to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/31c5cc53be804d01beccc9fb274f811c 2024-11-14T02:45:17,318 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/d790e2cc603e42fe85eb24aa11c8be9f to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/d790e2cc603e42fe85eb24aa11c8be9f 2024-11-14T02:45:17,319 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/f5502f65a3b4473ebd5cbab6a9ebceaa to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/C/f5502f65a3b4473ebd5cbab6a9ebceaa 2024-11-14T02:45:17,323 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/recovered.edits/302.seqid to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0/recovered.edits/302.seqid 2024-11-14T02:45:17,324 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:17,324 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T02:45:17,325 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T02:45:17,326 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T02:45:17,330 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114006617d889a04c8193280215dc8f84ac_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114006617d889a04c8193280215dc8f84ac_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:17,333 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111403204f564ab54e59b8669350f37e7826_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111403204f564ab54e59b8669350f37e7826_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:17,335 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111410abccd37e014168b540f2dd9391039c_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111410abccd37e014168b540f2dd9391039c_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:17,336 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142e7c88664b6d44ef848d345305dbcddb_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142e7c88664b6d44ef848d345305dbcddb_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:17,338 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143ad1e9996d5542dda5b33a3d5bc1c8a7_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143ad1e9996d5542dda5b33a3d5bc1c8a7_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:17,339 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143ef9ecea8894420580b8686225ef5017_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143ef9ecea8894420580b8686225ef5017_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:17,340 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144847412cd2d24f4ea6bcd9b04c65b5a9_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144847412cd2d24f4ea6bcd9b04c65b5a9_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:17,341 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144d49e1b97483458b9215f8eae6535f64_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144d49e1b97483458b9215f8eae6535f64_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:17,343 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144fd1eb2998344e6f97ad2cc7955ffd4d_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144fd1eb2998344e6f97ad2cc7955ffd4d_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:17,344 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111469dd57fd436d48ec99e74bacee156616_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111469dd57fd436d48ec99e74bacee156616_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:17,345 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111474f94e5d304d425686d53a813ae3865d_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111474f94e5d304d425686d53a813ae3865d_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:17,346 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111483ba32954e454272bfbfd869815ee5ee_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111483ba32954e454272bfbfd869815ee5ee_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:17,348 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a969a3d2ba8749d88005780e9f7ca68c_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a969a3d2ba8749d88005780e9f7ca68c_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:17,349 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cf42ac87835641e6bbcab7f8283b1e25_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cf42ac87835641e6bbcab7f8283b1e25_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:17,350 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f31cd3547af4491e9241e4eabc0554ff_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f31cd3547af4491e9241e4eabc0554ff_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:17,352 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fdf189c54b1740ab9fcfd8b08607802e_025639fc1f987c76e816bd52e91790c0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fdf189c54b1740ab9fcfd8b08607802e_025639fc1f987c76e816bd52e91790c0 2024-11-14T02:45:17,352 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T02:45:17,355 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=113, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:17,357 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T02:45:17,359 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T02:45:17,360 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=113, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:17,360 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T02:45:17,360 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731552317360"}]},"ts":"9223372036854775807"} 2024-11-14T02:45:17,361 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T02:45:17,361 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 025639fc1f987c76e816bd52e91790c0, NAME => 'TestAcidGuarantees,,1731552284420.025639fc1f987c76e816bd52e91790c0.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T02:45:17,361 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T02:45:17,361 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731552317361"}]},"ts":"9223372036854775807"} 2024-11-14T02:45:17,363 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T02:45:17,363 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=113, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:17,364 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 77 msec 2024-11-14T02:45:17,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T02:45:17,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T02:45:17,394 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:17,405 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=238 (was 238), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=308 (was 297) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5289 (was 5321) 2024-11-14T02:45:17,415 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=238, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=308, ProcessCount=11, AvailableMemoryMB=5288 2024-11-14T02:45:17,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T02:45:17,417 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T02:45:17,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:17,420 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T02:45:17,420 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:17,420 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 114 2024-11-14T02:45:17,422 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T02:45:17,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-14T02:45:17,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742218_1394 (size=959) 2024-11-14T02:45:17,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-14T02:45:17,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-14T02:45:17,833 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b 2024-11-14T02:45:17,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742219_1395 (size=53) 2024-11-14T02:45:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-14T02:45:18,262 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:45:18,262 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7922b12c74446660b37b836220447ee3, disabling compactions & flushes 2024-11-14T02:45:18,263 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:18,263 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:18,263 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. after waiting 0 ms 2024-11-14T02:45:18,263 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:18,263 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:18,263 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7922b12c74446660b37b836220447ee3: Waiting for close lock at 1731552318262Disabling compacts and flushes for region at 1731552318262Disabling writes for close at 1731552318263 (+1 ms)Writing region close event to WAL at 1731552318263Closed at 1731552318263 2024-11-14T02:45:18,266 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T02:45:18,267 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731552318266"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731552318266"}]},"ts":"1731552318266"} 2024-11-14T02:45:18,270 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T02:45:18,271 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T02:45:18,271 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552318271"}]},"ts":"1731552318271"} 2024-11-14T02:45:18,273 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T02:45:18,273 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7922b12c74446660b37b836220447ee3, ASSIGN}] 2024-11-14T02:45:18,275 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7922b12c74446660b37b836220447ee3, ASSIGN 2024-11-14T02:45:18,276 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7922b12c74446660b37b836220447ee3, ASSIGN; state=OFFLINE, location=83c5a5c119d5,41455,1731552191259; forceNewPlan=false, retain=false 2024-11-14T02:45:18,427 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=7922b12c74446660b37b836220447ee3, regionState=OPENING, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:18,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7922b12c74446660b37b836220447ee3, ASSIGN because future has completed 2024-11-14T02:45:18,432 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259}] 2024-11-14T02:45:18,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-14T02:45:18,593 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:18,593 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7752): Opening region: {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:45:18,596 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:18,597 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:45:18,597 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7794): checking encryption for 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:18,597 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7797): checking classloading for 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:18,599 INFO [StoreOpener-7922b12c74446660b37b836220447ee3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:18,600 INFO [StoreOpener-7922b12c74446660b37b836220447ee3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7922b12c74446660b37b836220447ee3 columnFamilyName A 2024-11-14T02:45:18,600 DEBUG [StoreOpener-7922b12c74446660b37b836220447ee3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:18,601 INFO [StoreOpener-7922b12c74446660b37b836220447ee3-1 {}] regionserver.HStore(327): Store=7922b12c74446660b37b836220447ee3/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:45:18,601 INFO [StoreOpener-7922b12c74446660b37b836220447ee3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:18,602 INFO [StoreOpener-7922b12c74446660b37b836220447ee3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7922b12c74446660b37b836220447ee3 columnFamilyName B 2024-11-14T02:45:18,602 DEBUG [StoreOpener-7922b12c74446660b37b836220447ee3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:18,603 INFO [StoreOpener-7922b12c74446660b37b836220447ee3-1 {}] regionserver.HStore(327): Store=7922b12c74446660b37b836220447ee3/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:45:18,603 INFO [StoreOpener-7922b12c74446660b37b836220447ee3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:18,604 INFO [StoreOpener-7922b12c74446660b37b836220447ee3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7922b12c74446660b37b836220447ee3 columnFamilyName C 2024-11-14T02:45:18,604 DEBUG [StoreOpener-7922b12c74446660b37b836220447ee3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:18,605 INFO [StoreOpener-7922b12c74446660b37b836220447ee3-1 {}] regionserver.HStore(327): Store=7922b12c74446660b37b836220447ee3/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:45:18,605 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1038): replaying wal for 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:18,606 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3 2024-11-14T02:45:18,606 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3 2024-11-14T02:45:18,607 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1048): stopping wal replay for 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:18,607 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1060): Cleaning up temporary data for 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:18,608 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T02:45:18,609 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1093): writing seq id for 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:18,611 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T02:45:18,612 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1114): Opened 7922b12c74446660b37b836220447ee3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59074861, jitterRate=-0.11971597373485565}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T02:45:18,612 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:18,613 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1006): Region open journal for 7922b12c74446660b37b836220447ee3: Running coprocessor pre-open hook at 1731552318597Writing region info on filesystem at 1731552318597Initializing all the Stores at 1731552318598 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552318598Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552318599 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552318599Cleaning up temporary data from old regions at 1731552318607 (+8 ms)Running coprocessor post-open hooks at 1731552318612 (+5 ms)Region opened successfully at 1731552318613 (+1 ms) 2024-11-14T02:45:18,614 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., pid=116, masterSystemTime=1731552318586 2024-11-14T02:45:18,616 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:18,616 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:18,617 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=7922b12c74446660b37b836220447ee3, regionState=OPEN, openSeqNum=2, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:18,618 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 because future has completed 2024-11-14T02:45:18,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-14T02:45:18,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; OpenRegionProcedure 7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 in 187 msec 2024-11-14T02:45:18,624 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-14T02:45:18,624 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7922b12c74446660b37b836220447ee3, ASSIGN in 349 msec 2024-11-14T02:45:18,625 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T02:45:18,625 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552318625"}]},"ts":"1731552318625"} 2024-11-14T02:45:18,627 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T02:45:18,628 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T02:45:18,629 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2110 sec 2024-11-14T02:45:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-14T02:45:19,585 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:19,588 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28ad78a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,588 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:45:19,588 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:19,590 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:45:19,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:19,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:45:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5aa54626, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:45:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:19,593 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:19,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4354eaa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,594 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:19,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:45:19,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:19,597 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:19,597 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T02:45:19,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 83c5a5c119d5,43239,1731552190584 2024-11-14T02:45:19,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@f48f4dd 2024-11-14T02:45:19,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T02:45:19,599 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T02:45:19,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,600 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:45:19,601 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:19,601 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:45:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:45:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41f0b23f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:45:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:19,602 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@394c1444, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,604 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:45:19,604 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:19,604 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:45:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:45:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fd82767, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:45:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:19,606 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:19,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@272a23b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,607 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,607 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:45:19,607 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:19,608 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:45:19,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:19,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:45:19,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a12a261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:45:19,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:19,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:19,609 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:19,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63896f80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,610 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,610 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:45:19,610 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:19,611 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:45:19,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:19,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:45:19,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72561f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:45:19,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:19,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:19,612 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:19,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a65eeef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7674f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,613 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:45:19,613 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:19,614 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:45:19,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:19,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:45:19,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9262201, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:45:19,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:19,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:19,615 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:19,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@164d35cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,617 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,617 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:45:19,617 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:19,618 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:45:19,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:19,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:45:19,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30992e20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:45:19,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:19,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:19,620 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@640b2bff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,622 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,622 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:45:19,622 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:19,623 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:45:19,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:19,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:45:19,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@674f6a05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:45:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:19,625 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3231341a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,627 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,627 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:45:19,627 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:19,628 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:45:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:45:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56693319, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:45:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:19,629 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:19,629 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:19,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@785adc97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,631 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,631 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:45:19,631 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:19,631 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:45:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:45:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e32b436, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:45:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:19,633 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:19,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b8b8dc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df5a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,634 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:45:19,634 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:19,635 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:45:19,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:19,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:45:19,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b8ecf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:45:19,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:19,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:19,637 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:19,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca3e2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:19,638 DEBUG [Thread-1799 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:19,638 DEBUG [Thread-1810 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:19,638 DEBUG [Thread-1803 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:19,638 DEBUG [Thread-1816 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:19,638 DEBUG [Thread-1805 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:19,639 DEBUG [Thread-1812 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:19,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:45:19,639 DEBUG [Thread-1814 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:19,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:45:19,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:45:19,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:19,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:19,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:45:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:45:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:19,640 DEBUG [Thread-1814 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:45:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:19,640 DEBUG [Thread-1797 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:45:19,640 DEBUG [Thread-1814 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:19,641 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:45:19,641 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:19,641 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:19,642 DEBUG [Thread-1808 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:19,642 DEBUG [Thread-1801 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:19,642 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:19,642 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:19,643 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:19,643 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:19,643 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:19,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:45:19,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:45:19,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:19,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:45:19,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:45:19,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:45:19,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:19,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:19,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:45:19,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:19,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:45:19,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-14T02:45:19,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:45:19,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:45:19,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:45:19,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T02:45:19,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:45:19,649 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:19,650 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:19,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2] 2024-11-14T02:45:19,651 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:19,651 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:19,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:19,654 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:45:19,655 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7922b12c74446660b37b836220447ee3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T02:45:19,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/25a90819454c44afb062d2738ec3a04a is 50, key is test_row_0/A:col10/1731552319653/Put/seqid=0 2024-11-14T02:45:19,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742220_1396 (size=16681) 2024-11-14T02:45:19,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/25a90819454c44afb062d2738ec3a04a 2024-11-14T02:45:19,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:19,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731552379701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:19,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:19,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/dfb2e52fed0d470eaf67be93fa68047a is 50, key is test_row_0/B:col10/1731552319653/Put/seqid=0 2024-11-14T02:45:19,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:19,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552379704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:19,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731552379706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:19,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:19,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552379708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:19,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:19,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552379707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:19,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742221_1397 (size=12001) 2024-11-14T02:45:19,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T02:45:19,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T02:45:19,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:19,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:19,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:19,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:19,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:19,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:19,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:19,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731552379813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:19,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:19,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:19,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552379823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:19,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:19,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:19,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731552379824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:19,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:19,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:19,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552379824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:19,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:19,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:19,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552379833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:19,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:19,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T02:45:19,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:19,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:19,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:19,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:19,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:19,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T02:45:20,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:20,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731552380023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:20,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:20,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:20,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552380034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:20,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:20,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731552380034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:20,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:20,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:20,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:20,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552380034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:20,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:20,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552380043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:20,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T02:45:20,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:20,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:20,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:20,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:20,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:20,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/dfb2e52fed0d470eaf67be93fa68047a 2024-11-14T02:45:20,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/ff219687502744e28eb807f8baaf76fe is 50, key is test_row_0/C:col10/1731552319653/Put/seqid=0 2024-11-14T02:45:20,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742222_1398 (size=12001) 2024-11-14T02:45:20,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T02:45:20,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:20,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:20,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:20,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:20,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:20,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:20,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T02:45:20,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:20,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731552380333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:20,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:20,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552380343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:20,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:20,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731552380344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:20,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552380344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:20,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552380353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:20,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:20,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T02:45:20,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:20,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:20,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:20,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:20,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:20,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/ff219687502744e28eb807f8baaf76fe 2024-11-14T02:45:20,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/25a90819454c44afb062d2738ec3a04a as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/25a90819454c44afb062d2738ec3a04a 2024-11-14T02:45:20,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/25a90819454c44afb062d2738ec3a04a, entries=250, sequenceid=12, filesize=16.3 K 2024-11-14T02:45:20,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/dfb2e52fed0d470eaf67be93fa68047a as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/dfb2e52fed0d470eaf67be93fa68047a 2024-11-14T02:45:20,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/dfb2e52fed0d470eaf67be93fa68047a, entries=150, sequenceid=12, filesize=11.7 K 2024-11-14T02:45:20,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T02:45:20,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:20,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:20,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:20,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/ff219687502744e28eb807f8baaf76fe as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ff219687502744e28eb807f8baaf76fe 2024-11-14T02:45:20,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:20,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:20,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:20,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ff219687502744e28eb807f8baaf76fe, entries=150, sequenceid=12, filesize=11.7 K 2024-11-14T02:45:20,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 7922b12c74446660b37b836220447ee3 in 918ms, sequenceid=12, compaction requested=false 2024-11-14T02:45:20,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:20,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T02:45:20,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:20,721 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 7922b12c74446660b37b836220447ee3 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T02:45:20,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/eccde65102a74122a7a38168b6f64729 is 50, key is test_row_0/A:col10/1731552319700/Put/seqid=0 2024-11-14T02:45:20,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742223_1399 (size=12001) 2024-11-14T02:45:20,729 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/eccde65102a74122a7a38168b6f64729 2024-11-14T02:45:20,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/848fcd76635c4d9a84e5095d9dbbb838 is 50, key is test_row_0/B:col10/1731552319700/Put/seqid=0 2024-11-14T02:45:20,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742224_1400 (size=12001) 2024-11-14T02:45:20,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T02:45:20,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:20,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:20,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:20,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731552380859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:20,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:20,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552380862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:20,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:20,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:20,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:20,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731552380862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:20,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:20,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:20,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552380863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:20,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:20,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:20,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552380863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:20,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:20,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731552380973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:20,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:20,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552380973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:20,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:20,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731552380973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:20,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:21,151 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/848fcd76635c4d9a84e5095d9dbbb838 2024-11-14T02:45:21,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/9ae05710bef3423babf92739ee5e7a30 is 50, key is test_row_0/C:col10/1731552319700/Put/seqid=0 2024-11-14T02:45:21,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742225_1401 (size=12001) 2024-11-14T02:45:21,161 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/9ae05710bef3423babf92739ee5e7a30 2024-11-14T02:45:21,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/eccde65102a74122a7a38168b6f64729 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/eccde65102a74122a7a38168b6f64729 2024-11-14T02:45:21,167 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/eccde65102a74122a7a38168b6f64729, entries=150, sequenceid=37, filesize=11.7 K 2024-11-14T02:45:21,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/848fcd76635c4d9a84e5095d9dbbb838 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/848fcd76635c4d9a84e5095d9dbbb838 2024-11-14T02:45:21,174 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/848fcd76635c4d9a84e5095d9dbbb838, entries=150, sequenceid=37, filesize=11.7 K 2024-11-14T02:45:21,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/9ae05710bef3423babf92739ee5e7a30 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/9ae05710bef3423babf92739ee5e7a30 2024-11-14T02:45:21,183 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/9ae05710bef3423babf92739ee5e7a30, entries=150, sequenceid=37, filesize=11.7 K 2024-11-14T02:45:21,184 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 7922b12c74446660b37b836220447ee3 in 462ms, sequenceid=37, compaction requested=false 2024-11-14T02:45:21,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:21,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:21,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-14T02:45:21,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-14T02:45:21,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:21,191 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:45:21,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-14T02:45:21,191 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7922b12c74446660b37b836220447ee3 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T02:45:21,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5350 sec 2024-11-14T02:45:21,193 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.5490 sec 2024-11-14T02:45:21,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/b2e19918caae4cf4ab76a6a02e52e398 is 50, key is test_row_0/A:col10/1731552320858/Put/seqid=0 2024-11-14T02:45:21,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742226_1402 (size=14341) 2024-11-14T02:45:21,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:21,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552381274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:21,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:21,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731552381275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:21,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:21,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:21,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:21,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731552381276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:21,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552381394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731552381394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:21,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731552381394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:21,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/b2e19918caae4cf4ab76a6a02e52e398 2024-11-14T02:45:21,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/67248d8af43b496d831ea69d08512a84 is 50, key is test_row_0/B:col10/1731552320858/Put/seqid=0 2024-11-14T02:45:21,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:21,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552381603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:21,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:21,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:21,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731552381603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:21,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:21,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:21,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731552381604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:21,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:21,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742227_1403 (size=12001) 2024-11-14T02:45:21,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T02:45:21,793 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:21,794 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-14T02:45:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T02:45:21,795 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:21,796 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:21,796 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:21,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552381884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:21,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552381884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:21,891 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_2' in 7922b12c74446660b37b836220447ee3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:45:21,891 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_2' in 7922b12c74446660b37b836220447ee3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:45:21,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:21,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T02:45:21,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552381923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731552381923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:21,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731552381923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:21,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T02:45:21,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:21,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:21,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:21,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:21,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/67248d8af43b496d831ea69d08512a84 2024-11-14T02:45:22,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/51df0d403aa844cdb4bab3dcba5bf1c6 is 50, key is test_row_0/C:col10/1731552320858/Put/seqid=0 2024-11-14T02:45:22,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742228_1404 (size=12001) 2024-11-14T02:45:22,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T02:45:22,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:22,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:22,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:22,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T02:45:22,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T02:45:22,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:22,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:22,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:22,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T02:45:22,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:22,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:22,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:22,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T02:45:22,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/51df0d403aa844cdb4bab3dcba5bf1c6 2024-11-14T02:45:22,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/b2e19918caae4cf4ab76a6a02e52e398 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/b2e19918caae4cf4ab76a6a02e52e398 2024-11-14T02:45:22,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/b2e19918caae4cf4ab76a6a02e52e398, entries=200, sequenceid=49, filesize=14.0 K 2024-11-14T02:45:22,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/67248d8af43b496d831ea69d08512a84 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/67248d8af43b496d831ea69d08512a84 2024-11-14T02:45:22,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/67248d8af43b496d831ea69d08512a84, entries=150, sequenceid=49, filesize=11.7 K 2024-11-14T02:45:22,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/51df0d403aa844cdb4bab3dcba5bf1c6 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/51df0d403aa844cdb4bab3dcba5bf1c6 2024-11-14T02:45:22,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731552382434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:22,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:22,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731552382434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:22,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:22,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/51df0d403aa844cdb4bab3dcba5bf1c6, entries=150, sequenceid=49, filesize=11.7 K 2024-11-14T02:45:22,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 7922b12c74446660b37b836220447ee3 in 1251ms, sequenceid=49, compaction requested=true 2024-11-14T02:45:22,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:22,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7922b12c74446660b37b836220447ee3:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:45:22,442 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:22,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:22,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7922b12c74446660b37b836220447ee3:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:45:22,442 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:22,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:22,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7922b12c74446660b37b836220447ee3:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:45:22,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:22,444 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:22,444 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:22,444 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 7922b12c74446660b37b836220447ee3/B is initiating minor compaction (all files) 2024-11-14T02:45:22,444 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 7922b12c74446660b37b836220447ee3/A is initiating minor compaction (all files) 2024-11-14T02:45:22,444 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7922b12c74446660b37b836220447ee3/A in TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:22,444 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7922b12c74446660b37b836220447ee3/B in TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:22,444 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/25a90819454c44afb062d2738ec3a04a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/eccde65102a74122a7a38168b6f64729, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/b2e19918caae4cf4ab76a6a02e52e398] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp, totalSize=42.0 K 2024-11-14T02:45:22,444 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/dfb2e52fed0d470eaf67be93fa68047a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/848fcd76635c4d9a84e5095d9dbbb838, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/67248d8af43b496d831ea69d08512a84] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp, totalSize=35.2 K 2024-11-14T02:45:22,444 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25a90819454c44afb062d2738ec3a04a, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731552319648 2024-11-14T02:45:22,444 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting dfb2e52fed0d470eaf67be93fa68047a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731552319648 2024-11-14T02:45:22,445 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting eccde65102a74122a7a38168b6f64729, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731552319700 2024-11-14T02:45:22,445 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 848fcd76635c4d9a84e5095d9dbbb838, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731552319700 2024-11-14T02:45:22,445 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2e19918caae4cf4ab76a6a02e52e398, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731552320858 2024-11-14T02:45:22,445 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 67248d8af43b496d831ea69d08512a84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731552320858 2024-11-14T02:45:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:22,453 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7922b12c74446660b37b836220447ee3#A#compaction#335 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:22,453 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7922b12c74446660b37b836220447ee3#B#compaction#336 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:22,453 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/24145443f4974e5ba44f3374f7f33033 is 50, key is test_row_0/A:col10/1731552320858/Put/seqid=0 2024-11-14T02:45:22,453 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/863ddf5d964c48f09de888aba8e4331c is 50, key is test_row_0/B:col10/1731552320858/Put/seqid=0 2024-11-14T02:45:22,454 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:45:22,454 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7922b12c74446660b37b836220447ee3 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T02:45:22,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742230_1406 (size=12104) 2024-11-14T02:45:22,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/917e6ed4160740ca90b9348c129bac42 is 50, key is test_row_0/A:col10/1731552321275/Put/seqid=0 2024-11-14T02:45:22,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742229_1405 (size=12104) 2024-11-14T02:45:22,467 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/863ddf5d964c48f09de888aba8e4331c as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/863ddf5d964c48f09de888aba8e4331c 2024-11-14T02:45:22,475 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7922b12c74446660b37b836220447ee3/B of 7922b12c74446660b37b836220447ee3 into 863ddf5d964c48f09de888aba8e4331c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:22,475 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:22,475 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., storeName=7922b12c74446660b37b836220447ee3/B, priority=13, startTime=1731552322442; duration=0sec 2024-11-14T02:45:22,475 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:22,475 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7922b12c74446660b37b836220447ee3:B 2024-11-14T02:45:22,476 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:22,476 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:22,476 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 7922b12c74446660b37b836220447ee3/C is initiating minor compaction (all files) 2024-11-14T02:45:22,476 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7922b12c74446660b37b836220447ee3/C in TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:22,477 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ff219687502744e28eb807f8baaf76fe, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/9ae05710bef3423babf92739ee5e7a30, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/51df0d403aa844cdb4bab3dcba5bf1c6] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp, totalSize=35.2 K 2024-11-14T02:45:22,477 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting ff219687502744e28eb807f8baaf76fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731552319648 2024-11-14T02:45:22,477 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ae05710bef3423babf92739ee5e7a30, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731552319700 2024-11-14T02:45:22,477 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 51df0d403aa844cdb4bab3dcba5bf1c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731552320858 2024-11-14T02:45:22,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742231_1407 (size=14341) 2024-11-14T02:45:22,484 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7922b12c74446660b37b836220447ee3#C#compaction#338 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:22,484 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/e14249cbd6f44220a30e7128f81e9d7f is 50, key is test_row_0/C:col10/1731552320858/Put/seqid=0 2024-11-14T02:45:22,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742232_1408 (size=12104) 2024-11-14T02:45:22,495 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/e14249cbd6f44220a30e7128f81e9d7f as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/e14249cbd6f44220a30e7128f81e9d7f 2024-11-14T02:45:22,499 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7922b12c74446660b37b836220447ee3/C of 7922b12c74446660b37b836220447ee3 into e14249cbd6f44220a30e7128f81e9d7f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:22,499 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:22,499 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., storeName=7922b12c74446660b37b836220447ee3/C, priority=13, startTime=1731552322442; duration=0sec 2024-11-14T02:45:22,499 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:22,500 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7922b12c74446660b37b836220447ee3:C 2024-11-14T02:45:22,524 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T02:45:22,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552382535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:22,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:22,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T02:45:22,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:22,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:22,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:22,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552382654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:22,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:22,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T02:45:22,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:22,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:22,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:22,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T02:45:22,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:22,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:22,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:22,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,871 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/24145443f4974e5ba44f3374f7f33033 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/24145443f4974e5ba44f3374f7f33033 2024-11-14T02:45:22,875 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7922b12c74446660b37b836220447ee3/A of 7922b12c74446660b37b836220447ee3 into 24145443f4974e5ba44f3374f7f33033(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:22,875 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:22,875 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., storeName=7922b12c74446660b37b836220447ee3/A, priority=13, startTime=1731552322442; duration=0sec 2024-11-14T02:45:22,875 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:22,875 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7922b12c74446660b37b836220447ee3:A 2024-11-14T02:45:22,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552382874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:22,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:22,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/917e6ed4160740ca90b9348c129bac42 2024-11-14T02:45:22,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/eb5e200812d04692a329768018c4036b is 50, key is test_row_0/B:col10/1731552321275/Put/seqid=0 2024-11-14T02:45:22,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742233_1409 (size=12001) 2024-11-14T02:45:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T02:45:23,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T02:45:23,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:23,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:23,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:23,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:23,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:23,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T02:45:23,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:23,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:23,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:23,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:23,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:23,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552383183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:23,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:23,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/eb5e200812d04692a329768018c4036b 2024-11-14T02:45:23,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/8880cb3af9db4f47b72fc3f296c49dd3 is 50, key is test_row_0/C:col10/1731552321275/Put/seqid=0 2024-11-14T02:45:23,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742234_1410 (size=12001) 2024-11-14T02:45:23,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T02:45:23,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:23,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:23,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:23,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:23,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:23,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:23,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731552383453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:23,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:23,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731552383453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:23,460 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_2' in 7922b12c74446660b37b836220447ee3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:45:23,460 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_2' in 7922b12c74446660b37b836220447ee3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:45:23,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:23,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:23,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T02:45:23,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:23,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:23,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:23,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:23,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:23,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:23,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T02:45:23,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:23,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:23,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:23,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:23,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:23,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:23,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:23,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552383693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:23,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:23,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/8880cb3af9db4f47b72fc3f296c49dd3 2024-11-14T02:45:23,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/917e6ed4160740ca90b9348c129bac42 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/917e6ed4160740ca90b9348c129bac42 2024-11-14T02:45:23,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/917e6ed4160740ca90b9348c129bac42, entries=200, sequenceid=74, filesize=14.0 K 2024-11-14T02:45:23,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/eb5e200812d04692a329768018c4036b as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/eb5e200812d04692a329768018c4036b 2024-11-14T02:45:23,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/eb5e200812d04692a329768018c4036b, entries=150, sequenceid=74, filesize=11.7 K 2024-11-14T02:45:23,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/8880cb3af9db4f47b72fc3f296c49dd3 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/8880cb3af9db4f47b72fc3f296c49dd3 2024-11-14T02:45:23,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/8880cb3af9db4f47b72fc3f296c49dd3, entries=150, sequenceid=74, filesize=11.7 K 2024-11-14T02:45:23,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 7922b12c74446660b37b836220447ee3 in 1263ms, sequenceid=74, compaction requested=false 2024-11-14T02:45:23,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:23,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T02:45:23,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:23,780 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 7922b12c74446660b37b836220447ee3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T02:45:23,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/0c3b9932e8b046d6bb04500ffe8dcc79 is 50, key is test_row_0/A:col10/1731552322515/Put/seqid=0 2024-11-14T02:45:23,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742235_1411 (size=12001) 2024-11-14T02:45:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:23,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T02:45:23,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552383988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:23,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:24,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552383995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:24,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:24,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552384103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:24,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:24,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552384103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:24,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:24,188 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/0c3b9932e8b046d6bb04500ffe8dcc79 2024-11-14T02:45:24,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/49637c0dde374657b45247e70da05c95 is 50, key is test_row_0/B:col10/1731552322515/Put/seqid=0 2024-11-14T02:45:24,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742236_1412 (size=12001) 2024-11-14T02:45:24,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:24,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552384313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:24,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:24,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:24,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552384313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:24,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:24,599 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/49637c0dde374657b45247e70da05c95 2024-11-14T02:45:24,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/c5790af34c264a10a943e34be0043503 is 50, key is test_row_0/C:col10/1731552322515/Put/seqid=0 2024-11-14T02:45:24,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742237_1413 (size=12001) 2024-11-14T02:45:24,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552384623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:24,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:24,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552384623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:24,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:24,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552384713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:24,722 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_1' in 7922b12c74446660b37b836220447ee3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:45:24,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:25,007 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/c5790af34c264a10a943e34be0043503 2024-11-14T02:45:25,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/0c3b9932e8b046d6bb04500ffe8dcc79 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/0c3b9932e8b046d6bb04500ffe8dcc79 2024-11-14T02:45:25,014 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/0c3b9932e8b046d6bb04500ffe8dcc79, entries=150, sequenceid=88, filesize=11.7 K 2024-11-14T02:45:25,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/49637c0dde374657b45247e70da05c95 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/49637c0dde374657b45247e70da05c95 2024-11-14T02:45:25,019 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/49637c0dde374657b45247e70da05c95, entries=150, sequenceid=88, filesize=11.7 K 2024-11-14T02:45:25,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/c5790af34c264a10a943e34be0043503 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/c5790af34c264a10a943e34be0043503 2024-11-14T02:45:25,022 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/c5790af34c264a10a943e34be0043503, entries=150, sequenceid=88, filesize=11.7 K 2024-11-14T02:45:25,023 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 7922b12c74446660b37b836220447ee3 in 1243ms, sequenceid=88, compaction requested=true 2024-11-14T02:45:25,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:25,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:25,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-14T02:45:25,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-14T02:45:25,026 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-14T02:45:25,026 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2280 sec 2024-11-14T02:45:25,027 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 3.2320 sec 2024-11-14T02:45:25,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:25,139 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:45:25,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7922b12c74446660b37b836220447ee3 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-14T02:45:25,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/a00c545f794c42a78dcf63c5cee75be0 is 50, key is test_row_0/A:col10/1731552325137/Put/seqid=0 2024-11-14T02:45:25,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742238_1414 (size=14341) 2024-11-14T02:45:25,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/a00c545f794c42a78dcf63c5cee75be0 2024-11-14T02:45:25,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/6dc2565424b24fc09cebd5e2ffc33404 is 50, key is test_row_0/B:col10/1731552325137/Put/seqid=0 2024-11-14T02:45:25,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742239_1415 (size=12001) 2024-11-14T02:45:25,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:25,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552385161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:25,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:25,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:25,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552385170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:25,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:25,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:25,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552385273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:25,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:25,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:25,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552385274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:25,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:25,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731552385473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:25,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731552385473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:25,481 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_2' in 7922b12c74446660b37b836220447ee3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:45:25,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,481 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_2' in 7922b12c74446660b37b836220447ee3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:45:25,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:25,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:25,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552385483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:25,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:25,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:25,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552385493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:25,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:25,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/6dc2565424b24fc09cebd5e2ffc33404 2024-11-14T02:45:25,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/ddda86b7c2d34034abed79792d3bef58 is 50, key is test_row_0/C:col10/1731552325137/Put/seqid=0 2024-11-14T02:45:25,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742240_1416 (size=12001) 2024-11-14T02:45:25,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:25,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552385794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:25,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:25,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:25,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552385803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:25,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:25,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T02:45:25,953 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:25,954 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:25,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-14T02:45:25,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T02:45:25,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/ddda86b7c2d34034abed79792d3bef58 2024-11-14T02:45:25,985 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:25,986 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:25,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:25,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/a00c545f794c42a78dcf63c5cee75be0 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/a00c545f794c42a78dcf63c5cee75be0 2024-11-14T02:45:25,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/a00c545f794c42a78dcf63c5cee75be0, entries=200, sequenceid=114, filesize=14.0 K 2024-11-14T02:45:25,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/6dc2565424b24fc09cebd5e2ffc33404 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/6dc2565424b24fc09cebd5e2ffc33404 2024-11-14T02:45:25,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/6dc2565424b24fc09cebd5e2ffc33404, entries=150, sequenceid=114, filesize=11.7 K 2024-11-14T02:45:25,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/ddda86b7c2d34034abed79792d3bef58 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ddda86b7c2d34034abed79792d3bef58 2024-11-14T02:45:25,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ddda86b7c2d34034abed79792d3bef58, entries=150, sequenceid=114, filesize=11.7 K 2024-11-14T02:45:25,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 7922b12c74446660b37b836220447ee3 in 860ms, sequenceid=114, compaction requested=true 2024-11-14T02:45:25,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:25,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7922b12c74446660b37b836220447ee3:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:45:25,999 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:45:25,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:25,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7922b12c74446660b37b836220447ee3:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:45:25,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:25,999 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:45:26,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7922b12c74446660b37b836220447ee3:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:45:26,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:26,000 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:45:26,000 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 7922b12c74446660b37b836220447ee3/B is initiating minor compaction (all files) 2024-11-14T02:45:26,000 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7922b12c74446660b37b836220447ee3/B in TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:26,000 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/863ddf5d964c48f09de888aba8e4331c, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/eb5e200812d04692a329768018c4036b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/49637c0dde374657b45247e70da05c95, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/6dc2565424b24fc09cebd5e2ffc33404] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp, totalSize=47.0 K 2024-11-14T02:45:26,001 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 863ddf5d964c48f09de888aba8e4331c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731552320858 2024-11-14T02:45:26,001 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:45:26,001 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 7922b12c74446660b37b836220447ee3/A is initiating minor compaction (all files) 2024-11-14T02:45:26,001 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7922b12c74446660b37b836220447ee3/A in TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:26,001 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/24145443f4974e5ba44f3374f7f33033, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/917e6ed4160740ca90b9348c129bac42, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/0c3b9932e8b046d6bb04500ffe8dcc79, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/a00c545f794c42a78dcf63c5cee75be0] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp, totalSize=51.5 K 2024-11-14T02:45:26,001 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24145443f4974e5ba44f3374f7f33033, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731552320858 2024-11-14T02:45:26,001 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting eb5e200812d04692a329768018c4036b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731552321263 2024-11-14T02:45:26,002 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 917e6ed4160740ca90b9348c129bac42, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731552321263 2024-11-14T02:45:26,002 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 49637c0dde374657b45247e70da05c95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731552322460 2024-11-14T02:45:26,002 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dc2565424b24fc09cebd5e2ffc33404, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731552323967 2024-11-14T02:45:26,002 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c3b9932e8b046d6bb04500ffe8dcc79, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731552322460 2024-11-14T02:45:26,003 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting a00c545f794c42a78dcf63c5cee75be0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731552323967 2024-11-14T02:45:26,009 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7922b12c74446660b37b836220447ee3#B#compaction#347 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:26,009 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/6819268912674f5eaaae72d5acee1860 is 50, key is test_row_0/B:col10/1731552325137/Put/seqid=0 2024-11-14T02:45:26,014 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7922b12c74446660b37b836220447ee3#A#compaction#348 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:26,014 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/039c350d92ee47129073a44e2775ec61 is 50, key is test_row_0/A:col10/1731552325137/Put/seqid=0 2024-11-14T02:45:26,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742241_1417 (size=12241) 2024-11-14T02:45:26,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742242_1418 (size=12241) 2024-11-14T02:45:26,044 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/039c350d92ee47129073a44e2775ec61 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/039c350d92ee47129073a44e2775ec61 2024-11-14T02:45:26,048 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7922b12c74446660b37b836220447ee3/A of 7922b12c74446660b37b836220447ee3 into 039c350d92ee47129073a44e2775ec61(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:26,048 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:26,048 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., storeName=7922b12c74446660b37b836220447ee3/A, priority=12, startTime=1731552325999; duration=0sec 2024-11-14T02:45:26,048 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:26,048 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7922b12c74446660b37b836220447ee3:A 2024-11-14T02:45:26,048 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:45:26,050 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:45:26,050 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 7922b12c74446660b37b836220447ee3/C is initiating minor compaction (all files) 2024-11-14T02:45:26,050 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7922b12c74446660b37b836220447ee3/C in TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:26,051 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/e14249cbd6f44220a30e7128f81e9d7f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/8880cb3af9db4f47b72fc3f296c49dd3, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/c5790af34c264a10a943e34be0043503, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ddda86b7c2d34034abed79792d3bef58] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp, totalSize=47.0 K 2024-11-14T02:45:26,052 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting e14249cbd6f44220a30e7128f81e9d7f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731552320858 2024-11-14T02:45:26,052 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8880cb3af9db4f47b72fc3f296c49dd3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731552321263 2024-11-14T02:45:26,052 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5790af34c264a10a943e34be0043503, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731552322460 2024-11-14T02:45:26,052 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddda86b7c2d34034abed79792d3bef58, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731552323967 2024-11-14T02:45:26,062 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7922b12c74446660b37b836220447ee3#C#compaction#349 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:26,063 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/b25fa1cef3d842d7b513a274b3b9fa43 is 50, key is test_row_0/C:col10/1731552325137/Put/seqid=0 2024-11-14T02:45:26,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T02:45:26,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742243_1419 (size=12241) 2024-11-14T02:45:26,076 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/b25fa1cef3d842d7b513a274b3b9fa43 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/b25fa1cef3d842d7b513a274b3b9fa43 2024-11-14T02:45:26,080 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7922b12c74446660b37b836220447ee3/C of 7922b12c74446660b37b836220447ee3 into b25fa1cef3d842d7b513a274b3b9fa43(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:26,080 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:26,080 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., storeName=7922b12c74446660b37b836220447ee3/C, priority=12, startTime=1731552325999; duration=0sec 2024-11-14T02:45:26,080 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:26,080 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7922b12c74446660b37b836220447ee3:C 2024-11-14T02:45:26,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T02:45:26,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:26,137 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 7922b12c74446660b37b836220447ee3 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-14T02:45:26,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/06b8efaf8638499cbfb6168d9f88125c is 50, key is test_row_0/A:col10/1731552325146/Put/seqid=0 2024-11-14T02:45:26,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742244_1420 (size=12001) 2024-11-14T02:45:26,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T02:45:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:26,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:26,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552386373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:26,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:26,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:26,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:26,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552386374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:26,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:26,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:26,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:26,426 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/6819268912674f5eaaae72d5acee1860 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/6819268912674f5eaaae72d5acee1860 2024-11-14T02:45:26,429 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7922b12c74446660b37b836220447ee3/B of 7922b12c74446660b37b836220447ee3 into 6819268912674f5eaaae72d5acee1860(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:26,429 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:26,429 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., storeName=7922b12c74446660b37b836220447ee3/B, priority=12, startTime=1731552325999; duration=0sec 2024-11-14T02:45:26,429 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:26,430 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7922b12c74446660b37b836220447ee3:B 2024-11-14T02:45:26,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552386484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:26,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552386493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:26,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:26,545 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/06b8efaf8638499cbfb6168d9f88125c 2024-11-14T02:45:26,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/e8d2c248b05d4db18b09eca7c4c267db is 50, key is test_row_0/B:col10/1731552325146/Put/seqid=0 2024-11-14T02:45:26,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742245_1421 (size=12001) 2024-11-14T02:45:26,555 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/e8d2c248b05d4db18b09eca7c4c267db 2024-11-14T02:45:26,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/3c5e6262858c4b82b777c12ba905600d is 50, key is test_row_0/C:col10/1731552325146/Put/seqid=0 2024-11-14T02:45:26,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742246_1422 (size=12001) 2024-11-14T02:45:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T02:45:26,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:26,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552386693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:26,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:26,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:26,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:26,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552386704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:26,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552386733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:26,739 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_1' in 7922b12c74446660b37b836220447ee3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:45:26,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:26,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:26,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:26,966 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/3c5e6262858c4b82b777c12ba905600d 2024-11-14T02:45:26,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/06b8efaf8638499cbfb6168d9f88125c as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/06b8efaf8638499cbfb6168d9f88125c 2024-11-14T02:45:26,971 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/06b8efaf8638499cbfb6168d9f88125c, entries=150, sequenceid=126, filesize=11.7 K 2024-11-14T02:45:26,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/e8d2c248b05d4db18b09eca7c4c267db as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/e8d2c248b05d4db18b09eca7c4c267db 2024-11-14T02:45:26,974 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/e8d2c248b05d4db18b09eca7c4c267db, entries=150, sequenceid=126, filesize=11.7 K 2024-11-14T02:45:26,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/3c5e6262858c4b82b777c12ba905600d as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/3c5e6262858c4b82b777c12ba905600d 2024-11-14T02:45:26,978 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/3c5e6262858c4b82b777c12ba905600d, entries=150, sequenceid=126, filesize=11.7 K 2024-11-14T02:45:26,979 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 7922b12c74446660b37b836220447ee3 in 842ms, sequenceid=126, compaction requested=false 2024-11-14T02:45:26,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:26,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:26,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-14T02:45:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-14T02:45:26,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-14T02:45:26,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 995 msec 2024-11-14T02:45:26,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.0290 sec 2024-11-14T02:45:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:27,013 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:45:27,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7922b12c74446660b37b836220447ee3 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-14T02:45:27,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/ac73b53cdefe428e9c8c33735b77e7d7 is 50, key is test_row_0/A:col10/1731552326367/Put/seqid=0 2024-11-14T02:45:27,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742247_1423 (size=14541) 2024-11-14T02:45:27,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552387032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:27,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:27,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552387032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:27,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T02:45:27,093 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:27,094 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-14T02:45:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T02:45:27,095 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:27,096 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:27,096 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:27,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552387143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552387144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:27,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T02:45:27,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T02:45:27,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:27,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:27,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:27,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:27,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:27,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:27,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552387353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:27,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552387353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:27,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T02:45:27,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:27,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:27,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:27,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:27,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:27,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:27,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T02:45:27,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/ac73b53cdefe428e9c8c33735b77e7d7 2024-11-14T02:45:27,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/c4513b5b81bc40c8969f67fb1f8f03e8 is 50, key is test_row_0/B:col10/1731552326367/Put/seqid=0 2024-11-14T02:45:27,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742248_1424 (size=12151) 2024-11-14T02:45:27,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T02:45:27,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:27,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:27,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:27,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:27,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:27,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552387664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552387664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:27,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T02:45:27,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:27,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:27,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:27,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:27,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:27,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T02:45:27,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/c4513b5b81bc40c8969f67fb1f8f03e8 2024-11-14T02:45:27,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/aa29140347704a5f930df48d5e4c8d8c is 50, key is test_row_0/C:col10/1731552326367/Put/seqid=0 2024-11-14T02:45:27,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742249_1425 (size=12151) 2024-11-14T02:45:27,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T02:45:27,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:27,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:27,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:27,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:27,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:28,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T02:45:28,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:28,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:28,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:28,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:28,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:28,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:28,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T02:45:28,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:28,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:28,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:28,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:28,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:28,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:28,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552388173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:28,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:28,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552388174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:28,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T02:45:28,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/aa29140347704a5f930df48d5e4c8d8c 2024-11-14T02:45:28,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/ac73b53cdefe428e9c8c33735b77e7d7 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/ac73b53cdefe428e9c8c33735b77e7d7 2024-11-14T02:45:28,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/ac73b53cdefe428e9c8c33735b77e7d7, entries=200, sequenceid=154, filesize=14.2 K 2024-11-14T02:45:28,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/c4513b5b81bc40c8969f67fb1f8f03e8 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/c4513b5b81bc40c8969f67fb1f8f03e8 2024-11-14T02:45:28,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/c4513b5b81bc40c8969f67fb1f8f03e8, entries=150, sequenceid=154, filesize=11.9 K 2024-11-14T02:45:28,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/aa29140347704a5f930df48d5e4c8d8c as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/aa29140347704a5f930df48d5e4c8d8c 2024-11-14T02:45:28,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/aa29140347704a5f930df48d5e4c8d8c, entries=150, sequenceid=154, filesize=11.9 K 2024-11-14T02:45:28,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 7922b12c74446660b37b836220447ee3 in 1244ms, sequenceid=154, compaction requested=true 2024-11-14T02:45:28,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:28,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7922b12c74446660b37b836220447ee3:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:45:28,257 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:28,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:28,258 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:28,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7922b12c74446660b37b836220447ee3:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:45:28,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:28,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7922b12c74446660b37b836220447ee3:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:45:28,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:28,258 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:28,258 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 7922b12c74446660b37b836220447ee3/A is initiating minor compaction (all files) 2024-11-14T02:45:28,258 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7922b12c74446660b37b836220447ee3/A in TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:28,258 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:28,258 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 7922b12c74446660b37b836220447ee3/B is initiating minor compaction (all files) 2024-11-14T02:45:28,258 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/039c350d92ee47129073a44e2775ec61, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/06b8efaf8638499cbfb6168d9f88125c, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/ac73b53cdefe428e9c8c33735b77e7d7] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp, totalSize=37.9 K 2024-11-14T02:45:28,258 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7922b12c74446660b37b836220447ee3/B in TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:28,259 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/6819268912674f5eaaae72d5acee1860, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/e8d2c248b05d4db18b09eca7c4c267db, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/c4513b5b81bc40c8969f67fb1f8f03e8] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp, totalSize=35.5 K 2024-11-14T02:45:28,259 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 039c350d92ee47129073a44e2775ec61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731552323967 2024-11-14T02:45:28,259 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 6819268912674f5eaaae72d5acee1860, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731552323967 2024-11-14T02:45:28,259 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06b8efaf8638499cbfb6168d9f88125c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731552325146 2024-11-14T02:45:28,259 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting e8d2c248b05d4db18b09eca7c4c267db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731552325146 2024-11-14T02:45:28,259 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac73b53cdefe428e9c8c33735b77e7d7, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731552326367 2024-11-14T02:45:28,259 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting c4513b5b81bc40c8969f67fb1f8f03e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731552326367 2024-11-14T02:45:28,268 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7922b12c74446660b37b836220447ee3#B#compaction#356 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:28,268 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/305518e222674ce98b8d999d5671cfe1 is 50, key is test_row_0/B:col10/1731552326367/Put/seqid=0 2024-11-14T02:45:28,271 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7922b12c74446660b37b836220447ee3#A#compaction#357 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:28,271 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/f5a3fb56526743c7ae67d02fa868c6e0 is 50, key is test_row_0/A:col10/1731552326367/Put/seqid=0 2024-11-14T02:45:28,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742250_1426 (size=12493) 2024-11-14T02:45:28,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742251_1427 (size=12493) 2024-11-14T02:45:28,285 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/305518e222674ce98b8d999d5671cfe1 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/305518e222674ce98b8d999d5671cfe1 2024-11-14T02:45:28,289 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/f5a3fb56526743c7ae67d02fa868c6e0 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/f5a3fb56526743c7ae67d02fa868c6e0 2024-11-14T02:45:28,291 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7922b12c74446660b37b836220447ee3/B of 7922b12c74446660b37b836220447ee3 into 305518e222674ce98b8d999d5671cfe1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:28,291 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:28,291 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., storeName=7922b12c74446660b37b836220447ee3/B, priority=13, startTime=1731552328258; duration=0sec 2024-11-14T02:45:28,291 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:28,291 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7922b12c74446660b37b836220447ee3:B 2024-11-14T02:45:28,291 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:28,292 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:28,292 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 7922b12c74446660b37b836220447ee3/C is initiating minor compaction (all files) 2024-11-14T02:45:28,292 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7922b12c74446660b37b836220447ee3/C in TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:28,292 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/b25fa1cef3d842d7b513a274b3b9fa43, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/3c5e6262858c4b82b777c12ba905600d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/aa29140347704a5f930df48d5e4c8d8c] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp, totalSize=35.5 K 2024-11-14T02:45:28,293 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting b25fa1cef3d842d7b513a274b3b9fa43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731552323967 2024-11-14T02:45:28,294 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c5e6262858c4b82b777c12ba905600d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731552325146 2024-11-14T02:45:28,295 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting aa29140347704a5f930df48d5e4c8d8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731552326367 2024-11-14T02:45:28,296 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7922b12c74446660b37b836220447ee3/A of 7922b12c74446660b37b836220447ee3 into f5a3fb56526743c7ae67d02fa868c6e0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:28,296 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:28,296 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., storeName=7922b12c74446660b37b836220447ee3/A, priority=13, startTime=1731552328257; duration=0sec 2024-11-14T02:45:28,296 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:28,296 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7922b12c74446660b37b836220447ee3:A 2024-11-14T02:45:28,302 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7922b12c74446660b37b836220447ee3#C#compaction#358 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:28,303 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/050828cb6bb54f51831014804c75eefd is 50, key is test_row_0/C:col10/1731552326367/Put/seqid=0 2024-11-14T02:45:28,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742252_1428 (size=12493) 2024-11-14T02:45:28,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T02:45:28,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:28,314 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 7922b12c74446660b37b836220447ee3 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-14T02:45:28,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/ef7e73d870694749849a813ee5a97e46 is 50, key is test_row_0/A:col10/1731552327031/Put/seqid=0 2024-11-14T02:45:28,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742253_1429 (size=12151) 2024-11-14T02:45:28,349 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/ef7e73d870694749849a813ee5a97e46 2024-11-14T02:45:28,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/7b03d8dbeeb54634a9cfd20f9028694f is 50, key is test_row_0/B:col10/1731552327031/Put/seqid=0 2024-11-14T02:45:28,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742254_1430 (size=12151) 2024-11-14T02:45:28,709 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/050828cb6bb54f51831014804c75eefd as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/050828cb6bb54f51831014804c75eefd 2024-11-14T02:45:28,712 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7922b12c74446660b37b836220447ee3/C of 7922b12c74446660b37b836220447ee3 into 050828cb6bb54f51831014804c75eefd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:28,712 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:28,712 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., storeName=7922b12c74446660b37b836220447ee3/C, priority=13, startTime=1731552328258; duration=0sec 2024-11-14T02:45:28,712 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:28,713 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7922b12c74446660b37b836220447ee3:C 2024-11-14T02:45:28,769 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/7b03d8dbeeb54634a9cfd20f9028694f 2024-11-14T02:45:28,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/e336a7721f2b449da0656b60c0660bca is 50, key is test_row_0/C:col10/1731552327031/Put/seqid=0 2024-11-14T02:45:28,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742255_1431 (size=12151) 2024-11-14T02:45:29,179 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/e336a7721f2b449da0656b60c0660bca 2024-11-14T02:45:29,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/ef7e73d870694749849a813ee5a97e46 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/ef7e73d870694749849a813ee5a97e46 2024-11-14T02:45:29,185 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/ef7e73d870694749849a813ee5a97e46, entries=150, sequenceid=165, filesize=11.9 K 2024-11-14T02:45:29,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/7b03d8dbeeb54634a9cfd20f9028694f as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/7b03d8dbeeb54634a9cfd20f9028694f 2024-11-14T02:45:29,189 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/7b03d8dbeeb54634a9cfd20f9028694f, entries=150, sequenceid=165, filesize=11.9 K 2024-11-14T02:45:29,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/e336a7721f2b449da0656b60c0660bca as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/e336a7721f2b449da0656b60c0660bca 2024-11-14T02:45:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:29,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:29,192 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/e336a7721f2b449da0656b60c0660bca, entries=150, sequenceid=165, filesize=11.9 K 2024-11-14T02:45:29,193 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=13.42 KB/13740 for 7922b12c74446660b37b836220447ee3 in 879ms, sequenceid=165, compaction requested=false 2024-11-14T02:45:29,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:29,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:29,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-14T02:45:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-14T02:45:29,195 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-14T02:45:29,195 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0980 sec 2024-11-14T02:45:29,196 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.1020 sec 2024-11-14T02:45:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:29,203 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:45:29,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7922b12c74446660b37b836220447ee3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T02:45:29,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/a11017f1f41949bda750ad26a042579d is 50, key is test_row_0/A:col10/1731552329188/Put/seqid=0 2024-11-14T02:45:29,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742256_1432 (size=16931) 2024-11-14T02:45:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T02:45:29,243 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:29,244 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-14T02:45:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T02:45:29,245 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:29,245 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:29,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:29,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552389266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:29,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552389267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:29,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T02:45:29,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552389373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:29,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:29,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552389383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:29,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:29,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T02:45:29,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:29,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:29,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:29,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:29,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:29,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731552389493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:29,500 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_2' in 7922b12c74446660b37b836220447ee3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:45:29,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:29,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731552389503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:29,511 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_2' in 7922b12c74446660b37b836220447ee3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:45:29,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:29,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T02:45:29,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:29,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:29,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:29,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:29,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T02:45:29,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552389583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:29,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:29,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552389593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:29,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:29,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/a11017f1f41949bda750ad26a042579d 2024-11-14T02:45:29,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/8dc5226adb46433b857ad2c44bd3eb04 is 50, key is test_row_0/B:col10/1731552329188/Put/seqid=0 2024-11-14T02:45:29,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742257_1433 (size=12151) 2024-11-14T02:45:29,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T02:45:29,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:29,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:29,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:29,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:29,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:29,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:29,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T02:45:29,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:29,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:29,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:29,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:29,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:29,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:29,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T02:45:29,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:29,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552389893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:29,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:29,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:29,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552389903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:29,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:30,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T02:45:30,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:30,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:30,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:30,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:30,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:30,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:30,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/8dc5226adb46433b857ad2c44bd3eb04 2024-11-14T02:45:30,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/4ea1709fdd5f4fcfb17d215309d0603a is 50, key is test_row_0/C:col10/1731552329188/Put/seqid=0 2024-11-14T02:45:30,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742258_1434 (size=12151) 2024-11-14T02:45:30,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T02:45:30,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:30,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:30,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:30,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:30,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:30,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:30,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T02:45:30,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:30,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:30,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:30,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:30,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:30,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:30,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T02:45:30,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:30,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552390413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:30,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:30,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:30,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:30,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552390413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:30,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/4ea1709fdd5f4fcfb17d215309d0603a 2024-11-14T02:45:30,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/a11017f1f41949bda750ad26a042579d as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/a11017f1f41949bda750ad26a042579d 2024-11-14T02:45:30,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/a11017f1f41949bda750ad26a042579d, entries=250, sequenceid=177, filesize=16.5 K 2024-11-14T02:45:30,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/8dc5226adb46433b857ad2c44bd3eb04 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/8dc5226adb46433b857ad2c44bd3eb04 2024-11-14T02:45:30,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/8dc5226adb46433b857ad2c44bd3eb04, entries=150, sequenceid=177, filesize=11.9 K 2024-11-14T02:45:30,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/4ea1709fdd5f4fcfb17d215309d0603a as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/4ea1709fdd5f4fcfb17d215309d0603a 2024-11-14T02:45:30,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/4ea1709fdd5f4fcfb17d215309d0603a, entries=150, sequenceid=177, filesize=11.9 K 2024-11-14T02:45:30,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 7922b12c74446660b37b836220447ee3 in 1241ms, sequenceid=177, compaction requested=true 2024-11-14T02:45:30,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:30,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7922b12c74446660b37b836220447ee3:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:45:30,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:30,444 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:30,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7922b12c74446660b37b836220447ee3:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:45:30,444 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:30,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:30,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7922b12c74446660b37b836220447ee3:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:45:30,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:30,445 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:30,445 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 7922b12c74446660b37b836220447ee3/A is initiating minor compaction (all files) 2024-11-14T02:45:30,445 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7922b12c74446660b37b836220447ee3/A in TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:30,445 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:30,445 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/f5a3fb56526743c7ae67d02fa868c6e0, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/ef7e73d870694749849a813ee5a97e46, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/a11017f1f41949bda750ad26a042579d] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp, totalSize=40.6 K 2024-11-14T02:45:30,445 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 7922b12c74446660b37b836220447ee3/B is initiating minor compaction (all files) 2024-11-14T02:45:30,445 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7922b12c74446660b37b836220447ee3/B in TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:30,445 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/305518e222674ce98b8d999d5671cfe1, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/7b03d8dbeeb54634a9cfd20f9028694f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/8dc5226adb46433b857ad2c44bd3eb04] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp, totalSize=35.9 K 2024-11-14T02:45:30,446 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5a3fb56526743c7ae67d02fa868c6e0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731552326367 2024-11-14T02:45:30,446 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 305518e222674ce98b8d999d5671cfe1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731552326367 2024-11-14T02:45:30,446 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef7e73d870694749849a813ee5a97e46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731552327023 2024-11-14T02:45:30,446 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b03d8dbeeb54634a9cfd20f9028694f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731552327023 2024-11-14T02:45:30,446 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting a11017f1f41949bda750ad26a042579d, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731552329188 2024-11-14T02:45:30,447 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dc5226adb46433b857ad2c44bd3eb04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731552329188 2024-11-14T02:45:30,453 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7922b12c74446660b37b836220447ee3#B#compaction#365 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:30,454 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/edf2de2feea147df8e9207fa35a57bcc is 50, key is test_row_0/B:col10/1731552329188/Put/seqid=0 2024-11-14T02:45:30,456 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7922b12c74446660b37b836220447ee3#A#compaction#366 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:30,456 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/06fdae0922774dea82ba774cf5c25be9 is 50, key is test_row_0/A:col10/1731552329188/Put/seqid=0 2024-11-14T02:45:30,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742259_1435 (size=12595) 2024-11-14T02:45:30,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742260_1436 (size=12595) 2024-11-14T02:45:30,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T02:45:30,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:30,465 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 7922b12c74446660b37b836220447ee3 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T02:45:30,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/85cb1ca169b74657aea51ed2199ca808 is 50, key is test_row_0/A:col10/1731552329264/Put/seqid=0 2024-11-14T02:45:30,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742261_1437 (size=12151) 2024-11-14T02:45:30,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:30,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:30,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:30,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552390815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:30,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:30,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:30,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:30,864 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/06fdae0922774dea82ba774cf5c25be9 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/06fdae0922774dea82ba774cf5c25be9 2024-11-14T02:45:30,864 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/edf2de2feea147df8e9207fa35a57bcc as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/edf2de2feea147df8e9207fa35a57bcc 2024-11-14T02:45:30,868 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7922b12c74446660b37b836220447ee3/A of 7922b12c74446660b37b836220447ee3 into 06fdae0922774dea82ba774cf5c25be9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:30,868 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:30,868 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., storeName=7922b12c74446660b37b836220447ee3/A, priority=13, startTime=1731552330444; duration=0sec 2024-11-14T02:45:30,868 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:30,868 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7922b12c74446660b37b836220447ee3:A 2024-11-14T02:45:30,868 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:30,869 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:30,869 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 7922b12c74446660b37b836220447ee3/C is initiating minor compaction (all files) 2024-11-14T02:45:30,869 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7922b12c74446660b37b836220447ee3/C in TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:30,869 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/050828cb6bb54f51831014804c75eefd, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/e336a7721f2b449da0656b60c0660bca, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/4ea1709fdd5f4fcfb17d215309d0603a] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp, totalSize=35.9 K 2024-11-14T02:45:30,869 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 050828cb6bb54f51831014804c75eefd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731552326367 2024-11-14T02:45:30,870 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting e336a7721f2b449da0656b60c0660bca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731552327023 2024-11-14T02:45:30,870 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ea1709fdd5f4fcfb17d215309d0603a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731552329188 2024-11-14T02:45:30,870 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7922b12c74446660b37b836220447ee3/B of 7922b12c74446660b37b836220447ee3 into edf2de2feea147df8e9207fa35a57bcc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:30,870 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:30,870 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., storeName=7922b12c74446660b37b836220447ee3/B, priority=13, startTime=1731552330444; duration=0sec 2024-11-14T02:45:30,870 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:30,870 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7922b12c74446660b37b836220447ee3:B 2024-11-14T02:45:30,873 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/85cb1ca169b74657aea51ed2199ca808 2024-11-14T02:45:30,875 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7922b12c74446660b37b836220447ee3#C#compaction#368 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:30,876 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/02d867b2fdaf4e308d8377ea0329a693 is 50, key is test_row_0/C:col10/1731552329188/Put/seqid=0 2024-11-14T02:45:30,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/e59628e58fe54929b9703bffd2641cfb is 50, key is test_row_0/B:col10/1731552329264/Put/seqid=0 2024-11-14T02:45:30,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742262_1438 (size=12595) 2024-11-14T02:45:30,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742263_1439 (size=12151) 2024-11-14T02:45:30,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:30,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552390933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:30,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:30,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:30,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:31,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552391144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:31,281 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/e59628e58fe54929b9703bffd2641cfb 2024-11-14T02:45:31,294 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/02d867b2fdaf4e308d8377ea0329a693 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/02d867b2fdaf4e308d8377ea0329a693 2024-11-14T02:45:31,298 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7922b12c74446660b37b836220447ee3/C of 7922b12c74446660b37b836220447ee3 into 02d867b2fdaf4e308d8377ea0329a693(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:31,298 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:31,298 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., storeName=7922b12c74446660b37b836220447ee3/C, priority=13, startTime=1731552330444; duration=0sec 2024-11-14T02:45:31,298 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:31,299 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7922b12c74446660b37b836220447ee3:C 2024-11-14T02:45:31,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/8b3c413e0e00454ab8f5b257da287d8c is 50, key is test_row_0/C:col10/1731552329264/Put/seqid=0 2024-11-14T02:45:31,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742264_1440 (size=12151) 2024-11-14T02:45:31,310 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/8b3c413e0e00454ab8f5b257da287d8c 2024-11-14T02:45:31,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/85cb1ca169b74657aea51ed2199ca808 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/85cb1ca169b74657aea51ed2199ca808 2024-11-14T02:45:31,317 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/85cb1ca169b74657aea51ed2199ca808, entries=150, sequenceid=202, filesize=11.9 K 2024-11-14T02:45:31,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/e59628e58fe54929b9703bffd2641cfb as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/e59628e58fe54929b9703bffd2641cfb 2024-11-14T02:45:31,321 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/e59628e58fe54929b9703bffd2641cfb, entries=150, sequenceid=202, filesize=11.9 K 2024-11-14T02:45:31,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/8b3c413e0e00454ab8f5b257da287d8c as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/8b3c413e0e00454ab8f5b257da287d8c 2024-11-14T02:45:31,324 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/8b3c413e0e00454ab8f5b257da287d8c, entries=150, sequenceid=202, filesize=11.9 K 2024-11-14T02:45:31,325 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 7922b12c74446660b37b836220447ee3 in 860ms, sequenceid=202, compaction requested=false 2024-11-14T02:45:31,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:31,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:31,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-14T02:45:31,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-14T02:45:31,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-14T02:45:31,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0810 sec 2024-11-14T02:45:31,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.0840 sec 2024-11-14T02:45:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T02:45:31,394 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:31,394 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-14T02:45:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T02:45:31,396 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:31,396 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:31,396 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:31,426 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:45:31,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7922b12c74446660b37b836220447ee3 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T02:45:31,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/287f75944f4e4def87f766f57f300a74 is 50, key is test_row_0/A:col10/1731552331426/Put/seqid=0 2024-11-14T02:45:31,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742265_1441 (size=16931) 2024-11-14T02:45:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T02:45:31,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552391515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:31,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:31,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:31,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552391516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:31,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:31,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:31,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:31,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:31,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552391520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:31,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:31,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:31,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:31,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:31,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:31,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:31,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:31,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:31,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:31,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552391633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:31,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:31,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:31,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:31,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552391633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:31,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:31,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:31,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552391633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:31,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:31,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:31,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:31,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:31,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:31,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:31,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:31,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:31,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:31,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T02:45:31,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/287f75944f4e4def87f766f57f300a74 2024-11-14T02:45:31,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/667fc89c86a845bc91c2671718bae26c is 50, key is test_row_0/B:col10/1731552331426/Put/seqid=0 2024-11-14T02:45:31,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742266_1442 (size=12151) 2024-11-14T02:45:31,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:31,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552391844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:31,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:31,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:31,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:31,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:31,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552391844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:31,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:31,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:31,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:31,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/667fc89c86a845bc91c2671718bae26c 2024-11-14T02:45:31,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/2236343dec8d4840a54beeb157031be3 is 50, key is test_row_0/C:col10/1731552331426/Put/seqid=0 2024-11-14T02:45:31,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:31,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552391844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:31,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:31,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:31,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:31,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:31,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:31,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:31,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:31,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:31,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:31,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:31,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742267_1443 (size=12151) 2024-11-14T02:45:32,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:32,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:32,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:32,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:32,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:32,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:32,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:32,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T02:45:32,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:32,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:32,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:32,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:32,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:32,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552392153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:32,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:32,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:32,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:32,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552392153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:32,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:32,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:32,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552392164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:32,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:32,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:32,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:32,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/2236343dec8d4840a54beeb157031be3 2024-11-14T02:45:32,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/287f75944f4e4def87f766f57f300a74 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/287f75944f4e4def87f766f57f300a74 2024-11-14T02:45:32,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/287f75944f4e4def87f766f57f300a74, entries=250, sequenceid=218, filesize=16.5 K 2024-11-14T02:45:32,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/667fc89c86a845bc91c2671718bae26c as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/667fc89c86a845bc91c2671718bae26c 2024-11-14T02:45:32,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/667fc89c86a845bc91c2671718bae26c, entries=150, sequenceid=218, filesize=11.9 K 2024-11-14T02:45:32,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/2236343dec8d4840a54beeb157031be3 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/2236343dec8d4840a54beeb157031be3 2024-11-14T02:45:32,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/2236343dec8d4840a54beeb157031be3, entries=150, sequenceid=218, filesize=11.9 K 2024-11-14T02:45:32,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 7922b12c74446660b37b836220447ee3 in 848ms, sequenceid=218, compaction requested=true 2024-11-14T02:45:32,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:32,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7922b12c74446660b37b836220447ee3:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:45:32,274 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:32,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:32,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7922b12c74446660b37b836220447ee3:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:45:32,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:32,274 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:32,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7922b12c74446660b37b836220447ee3:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:45:32,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:32,275 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:32,275 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:32,275 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 7922b12c74446660b37b836220447ee3/B is initiating minor compaction (all files) 2024-11-14T02:45:32,275 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 7922b12c74446660b37b836220447ee3/A is initiating minor compaction (all files) 2024-11-14T02:45:32,275 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7922b12c74446660b37b836220447ee3/B in TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:32,275 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7922b12c74446660b37b836220447ee3/A in TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:32,275 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/edf2de2feea147df8e9207fa35a57bcc, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/e59628e58fe54929b9703bffd2641cfb, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/667fc89c86a845bc91c2671718bae26c] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp, totalSize=36.0 K 2024-11-14T02:45:32,275 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/06fdae0922774dea82ba774cf5c25be9, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/85cb1ca169b74657aea51ed2199ca808, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/287f75944f4e4def87f766f57f300a74] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp, totalSize=40.7 K 2024-11-14T02:45:32,276 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting edf2de2feea147df8e9207fa35a57bcc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731552329188 2024-11-14T02:45:32,276 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06fdae0922774dea82ba774cf5c25be9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731552329188 2024-11-14T02:45:32,276 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting e59628e58fe54929b9703bffd2641cfb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731552329254 2024-11-14T02:45:32,276 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85cb1ca169b74657aea51ed2199ca808, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731552329254 2024-11-14T02:45:32,276 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 287f75944f4e4def87f766f57f300a74, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731552330794 2024-11-14T02:45:32,276 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 667fc89c86a845bc91c2671718bae26c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731552330814 2024-11-14T02:45:32,281 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7922b12c74446660b37b836220447ee3#A#compaction#374 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:32,282 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/0f5b9ff81f5e42b49789a08e2be1ec38 is 50, key is test_row_0/A:col10/1731552331426/Put/seqid=0 2024-11-14T02:45:32,284 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7922b12c74446660b37b836220447ee3#B#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:32,284 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/8e6791f9902b4e518dd573e72cba9431 is 50, key is test_row_0/B:col10/1731552331426/Put/seqid=0 2024-11-14T02:45:32,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742268_1444 (size=12697) 2024-11-14T02:45:32,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742269_1445 (size=12697) 2024-11-14T02:45:32,293 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/0f5b9ff81f5e42b49789a08e2be1ec38 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/0f5b9ff81f5e42b49789a08e2be1ec38 2024-11-14T02:45:32,293 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/8e6791f9902b4e518dd573e72cba9431 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/8e6791f9902b4e518dd573e72cba9431 2024-11-14T02:45:32,296 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7922b12c74446660b37b836220447ee3/B of 7922b12c74446660b37b836220447ee3 into 8e6791f9902b4e518dd573e72cba9431(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:32,296 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7922b12c74446660b37b836220447ee3/A of 7922b12c74446660b37b836220447ee3 into 0f5b9ff81f5e42b49789a08e2be1ec38(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:32,296 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:32,296 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:32,297 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., storeName=7922b12c74446660b37b836220447ee3/B, priority=13, startTime=1731552332274; duration=0sec 2024-11-14T02:45:32,297 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., storeName=7922b12c74446660b37b836220447ee3/A, priority=13, startTime=1731552332274; duration=0sec 2024-11-14T02:45:32,297 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:32,297 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7922b12c74446660b37b836220447ee3:B 2024-11-14T02:45:32,297 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:32,297 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7922b12c74446660b37b836220447ee3:A 2024-11-14T02:45:32,297 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:32,297 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:32,297 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 7922b12c74446660b37b836220447ee3/C is initiating minor compaction (all files) 2024-11-14T02:45:32,297 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7922b12c74446660b37b836220447ee3/C in TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:32,297 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/02d867b2fdaf4e308d8377ea0329a693, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/8b3c413e0e00454ab8f5b257da287d8c, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/2236343dec8d4840a54beeb157031be3] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp, totalSize=36.0 K 2024-11-14T02:45:32,298 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 02d867b2fdaf4e308d8377ea0329a693, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731552329188 2024-11-14T02:45:32,298 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b3c413e0e00454ab8f5b257da287d8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731552329254 2024-11-14T02:45:32,298 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 2236343dec8d4840a54beeb157031be3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731552330814 2024-11-14T02:45:32,305 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7922b12c74446660b37b836220447ee3#C#compaction#376 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:32,306 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/900af1c8ea384250bba524d35b6ecb8b is 50, key is test_row_0/C:col10/1731552331426/Put/seqid=0 2024-11-14T02:45:32,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742270_1446 (size=12697) 2024-11-14T02:45:32,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:32,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:32,310 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 7922b12c74446660b37b836220447ee3 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-14T02:45:32,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/2bf5ee6e7a484ee2988533b676a1d377 is 50, key is test_row_0/A:col10/1731552331519/Put/seqid=0 2024-11-14T02:45:32,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742271_1447 (size=12151) 2024-11-14T02:45:32,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T02:45:32,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:32,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:32,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:32,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552392690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:32,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:32,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:32,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:32,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:32,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552392695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:32,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:32,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552392696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:32,713 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/900af1c8ea384250bba524d35b6ecb8b as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/900af1c8ea384250bba524d35b6ecb8b 2024-11-14T02:45:32,716 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7922b12c74446660b37b836220447ee3/C of 7922b12c74446660b37b836220447ee3 into 900af1c8ea384250bba524d35b6ecb8b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:32,716 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:32,716 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., storeName=7922b12c74446660b37b836220447ee3/C, priority=13, startTime=1731552332274; duration=0sec 2024-11-14T02:45:32,716 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:32,716 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7922b12c74446660b37b836220447ee3:C 2024-11-14T02:45:32,718 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/2bf5ee6e7a484ee2988533b676a1d377 2024-11-14T02:45:32,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/d9300bee19b04daabfaa8b463fd2edc0 is 50, key is test_row_0/B:col10/1731552331519/Put/seqid=0 2024-11-14T02:45:32,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742272_1448 (size=12151) 2024-11-14T02:45:32,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:32,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552392804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:32,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:32,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:32,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:32,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:32,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552392804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:32,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:32,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:32,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:32,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:32,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552392804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:32,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:32,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:32,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:33,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552393013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:33,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:33,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552393013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:33,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552393024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:33,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:33,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:33,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:33,126 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/d9300bee19b04daabfaa8b463fd2edc0 2024-11-14T02:45:33,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/54d84d7baa404de18a157abe7b31d79b is 50, key is test_row_0/C:col10/1731552331519/Put/seqid=0 2024-11-14T02:45:33,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742273_1449 (size=12151) 2024-11-14T02:45:33,136 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/54d84d7baa404de18a157abe7b31d79b 2024-11-14T02:45:33,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/2bf5ee6e7a484ee2988533b676a1d377 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/2bf5ee6e7a484ee2988533b676a1d377 2024-11-14T02:45:33,142 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/2bf5ee6e7a484ee2988533b676a1d377, entries=150, sequenceid=243, filesize=11.9 K 2024-11-14T02:45:33,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/d9300bee19b04daabfaa8b463fd2edc0 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/d9300bee19b04daabfaa8b463fd2edc0 2024-11-14T02:45:33,146 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/d9300bee19b04daabfaa8b463fd2edc0, entries=150, sequenceid=243, filesize=11.9 K 2024-11-14T02:45:33,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/54d84d7baa404de18a157abe7b31d79b as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/54d84d7baa404de18a157abe7b31d79b 2024-11-14T02:45:33,149 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/54d84d7baa404de18a157abe7b31d79b, entries=150, sequenceid=243, filesize=11.9 K 2024-11-14T02:45:33,150 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 7922b12c74446660b37b836220447ee3 in 840ms, sequenceid=243, compaction requested=false 2024-11-14T02:45:33,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:33,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:33,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-14T02:45:33,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-14T02:45:33,153 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-14T02:45:33,153 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7550 sec 2024-11-14T02:45:33,154 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.7590 sec 2024-11-14T02:45:33,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:33,327 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:45:33,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7922b12c74446660b37b836220447ee3 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-14T02:45:33,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/afaafc734ce14b7bae02f78bba12e2b8 is 50, key is test_row_0/A:col10/1731552332686/Put/seqid=0 2024-11-14T02:45:33,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742274_1450 (size=14591) 2024-11-14T02:45:33,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:33,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552393371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:33,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:33,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:33,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:33,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:33,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552393376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:33,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:33,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552393381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:33,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:33,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552393483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:33,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:33,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:33,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:33,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552393494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:33,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:33,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:33,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552393494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:33,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:33,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:33,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:33,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:33,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:33,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T02:45:33,543 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:33,544 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:33,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-14T02:45:33,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T02:45:33,545 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:33,546 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:33,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T02:45:33,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:33,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552393693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:33,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:33,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:33,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:33,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T02:45:33,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:33,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:33,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:33,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:33,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:33,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:33,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:33,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552393713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:33,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:33,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552393713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:33,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:33,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:33,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:33,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/afaafc734ce14b7bae02f78bba12e2b8 2024-11-14T02:45:33,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/4c1a551b4aea4883b7beb1126acd26bd is 50, key is test_row_0/B:col10/1731552332686/Put/seqid=0 2024-11-14T02:45:33,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742275_1451 (size=12201) 2024-11-14T02:45:33,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T02:45:33,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:33,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:33,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:33,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:33,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T02:45:34,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T02:45:34,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:34,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:34,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:34,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:34,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:34,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:34,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:34,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552394003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:34,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:34,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:34,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:34,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:34,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552394034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:34,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:34,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:34,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:34,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552394034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:34,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:34,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:34,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:34,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:34,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/4c1a551b4aea4883b7beb1126acd26bd 2024-11-14T02:45:34,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/ae04147a60164e069ef152f0531ae804 is 50, key is test_row_0/C:col10/1731552332686/Put/seqid=0 2024-11-14T02:45:34,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742276_1452 (size=12201) 2024-11-14T02:45:34,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T02:45:34,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:34,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:34,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:34,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:34,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:34,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:34,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T02:45:34,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T02:45:34,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:34,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:34,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:34,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:34,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:34,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:34,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T02:45:34,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:34,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:34,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:34,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:34,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:34,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:34,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:34,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552394513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:34,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:34,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:34,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:34,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:34,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552394543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:34,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:34,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:34,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:34,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:34,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552394544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:34,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:34,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:34,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:34,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/ae04147a60164e069ef152f0531ae804 2024-11-14T02:45:34,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/afaafc734ce14b7bae02f78bba12e2b8 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/afaafc734ce14b7bae02f78bba12e2b8 2024-11-14T02:45:34,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/afaafc734ce14b7bae02f78bba12e2b8, entries=200, sequenceid=258, filesize=14.2 K 2024-11-14T02:45:34,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/4c1a551b4aea4883b7beb1126acd26bd as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/4c1a551b4aea4883b7beb1126acd26bd 2024-11-14T02:45:34,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/4c1a551b4aea4883b7beb1126acd26bd, entries=150, sequenceid=258, filesize=11.9 K 2024-11-14T02:45:34,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/ae04147a60164e069ef152f0531ae804 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ae04147a60164e069ef152f0531ae804 2024-11-14T02:45:34,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ae04147a60164e069ef152f0531ae804, entries=150, sequenceid=258, filesize=11.9 K 2024-11-14T02:45:34,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 7922b12c74446660b37b836220447ee3 in 1239ms, sequenceid=258, compaction requested=true 2024-11-14T02:45:34,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:34,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7922b12c74446660b37b836220447ee3:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:45:34,566 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:34,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:34,566 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:34,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7922b12c74446660b37b836220447ee3:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:45:34,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:34,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7922b12c74446660b37b836220447ee3:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:45:34,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:34,567 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:34,567 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39439 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:34,567 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 7922b12c74446660b37b836220447ee3/A is initiating minor compaction (all files) 2024-11-14T02:45:34,567 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 7922b12c74446660b37b836220447ee3/B is initiating minor compaction (all files) 2024-11-14T02:45:34,567 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7922b12c74446660b37b836220447ee3/B in TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:34,567 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7922b12c74446660b37b836220447ee3/A in TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:34,567 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/8e6791f9902b4e518dd573e72cba9431, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/d9300bee19b04daabfaa8b463fd2edc0, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/4c1a551b4aea4883b7beb1126acd26bd] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp, totalSize=36.2 K 2024-11-14T02:45:34,567 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/0f5b9ff81f5e42b49789a08e2be1ec38, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/2bf5ee6e7a484ee2988533b676a1d377, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/afaafc734ce14b7bae02f78bba12e2b8] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp, totalSize=38.5 K 2024-11-14T02:45:34,567 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f5b9ff81f5e42b49789a08e2be1ec38, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731552330814 2024-11-14T02:45:34,567 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e6791f9902b4e518dd573e72cba9431, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731552330814 2024-11-14T02:45:34,568 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bf5ee6e7a484ee2988533b676a1d377, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731552331514 2024-11-14T02:45:34,568 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting d9300bee19b04daabfaa8b463fd2edc0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731552331514 2024-11-14T02:45:34,568 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting afaafc734ce14b7bae02f78bba12e2b8, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731552332686 2024-11-14T02:45:34,568 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c1a551b4aea4883b7beb1126acd26bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731552332686 2024-11-14T02:45:34,577 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7922b12c74446660b37b836220447ee3#A#compaction#383 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:34,578 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/fba174fccbb2458b8119076f2af553c1 is 50, key is test_row_0/A:col10/1731552332686/Put/seqid=0 2024-11-14T02:45:34,582 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7922b12c74446660b37b836220447ee3#B#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:34,582 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/6a0e0e65b016487f9a3934c011ccb014 is 50, key is test_row_0/B:col10/1731552332686/Put/seqid=0 2024-11-14T02:45:34,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742277_1453 (size=12849) 2024-11-14T02:45:34,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742278_1454 (size=12849) 2024-11-14T02:45:34,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T02:45:34,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:34,614 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 7922b12c74446660b37b836220447ee3 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-14T02:45:34,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/5026a4bf947f4e99a0ae9352d65adaca is 50, key is test_row_0/A:col10/1731552333379/Put/seqid=0 2024-11-14T02:45:34,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742279_1455 (size=12301) 2024-11-14T02:45:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T02:45:34,989 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/fba174fccbb2458b8119076f2af553c1 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/fba174fccbb2458b8119076f2af553c1 2024-11-14T02:45:34,990 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/6a0e0e65b016487f9a3934c011ccb014 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/6a0e0e65b016487f9a3934c011ccb014 2024-11-14T02:45:34,993 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7922b12c74446660b37b836220447ee3/A of 7922b12c74446660b37b836220447ee3 into fba174fccbb2458b8119076f2af553c1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:34,993 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:34,993 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., storeName=7922b12c74446660b37b836220447ee3/A, priority=13, startTime=1731552334566; duration=0sec 2024-11-14T02:45:34,993 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:34,993 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7922b12c74446660b37b836220447ee3:A 2024-11-14T02:45:34,993 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:34,994 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7922b12c74446660b37b836220447ee3/B of 7922b12c74446660b37b836220447ee3 into 6a0e0e65b016487f9a3934c011ccb014(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:34,994 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:34,994 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., storeName=7922b12c74446660b37b836220447ee3/B, priority=13, startTime=1731552334566; duration=0sec 2024-11-14T02:45:34,994 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:34,994 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:34,994 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7922b12c74446660b37b836220447ee3:B 2024-11-14T02:45:34,994 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 7922b12c74446660b37b836220447ee3/C is initiating minor compaction (all files) 2024-11-14T02:45:34,994 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7922b12c74446660b37b836220447ee3/C in TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:34,994 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/900af1c8ea384250bba524d35b6ecb8b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/54d84d7baa404de18a157abe7b31d79b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ae04147a60164e069ef152f0531ae804] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp, totalSize=36.2 K 2024-11-14T02:45:34,995 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 900af1c8ea384250bba524d35b6ecb8b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731552330814 2024-11-14T02:45:34,995 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54d84d7baa404de18a157abe7b31d79b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731552331514 2024-11-14T02:45:34,995 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae04147a60164e069ef152f0531ae804, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731552332686 2024-11-14T02:45:35,000 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7922b12c74446660b37b836220447ee3#C#compaction#386 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:35,000 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/345a6d6a69d843efb62604c4596a7aea is 50, key is test_row_0/C:col10/1731552332686/Put/seqid=0 2024-11-14T02:45:35,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742280_1456 (size=12849) 2024-11-14T02:45:35,023 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/5026a4bf947f4e99a0ae9352d65adaca 2024-11-14T02:45:35,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/60fe0493daba4cc5b89a9143256afdf5 is 50, key is test_row_0/B:col10/1731552333379/Put/seqid=0 2024-11-14T02:45:35,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742281_1457 (size=12301) 2024-11-14T02:45:35,034 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/60fe0493daba4cc5b89a9143256afdf5 2024-11-14T02:45:35,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/15d9411bb42a4fe296b5b5baa0af45ee is 50, key is test_row_0/C:col10/1731552333379/Put/seqid=0 2024-11-14T02:45:35,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742282_1458 (size=12301) 2024-11-14T02:45:35,406 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/345a6d6a69d843efb62604c4596a7aea as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/345a6d6a69d843efb62604c4596a7aea 2024-11-14T02:45:35,409 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7922b12c74446660b37b836220447ee3/C of 7922b12c74446660b37b836220447ee3 into 345a6d6a69d843efb62604c4596a7aea(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:35,409 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:35,409 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., storeName=7922b12c74446660b37b836220447ee3/C, priority=13, startTime=1731552334566; duration=0sec 2024-11-14T02:45:35,410 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:35,410 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7922b12c74446660b37b836220447ee3:C 2024-11-14T02:45:35,443 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/15d9411bb42a4fe296b5b5baa0af45ee 2024-11-14T02:45:35,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/5026a4bf947f4e99a0ae9352d65adaca as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/5026a4bf947f4e99a0ae9352d65adaca 2024-11-14T02:45:35,449 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/5026a4bf947f4e99a0ae9352d65adaca, entries=150, sequenceid=280, filesize=12.0 K 2024-11-14T02:45:35,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/60fe0493daba4cc5b89a9143256afdf5 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/60fe0493daba4cc5b89a9143256afdf5 2024-11-14T02:45:35,453 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/60fe0493daba4cc5b89a9143256afdf5, entries=150, sequenceid=280, filesize=12.0 K 2024-11-14T02:45:35,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/15d9411bb42a4fe296b5b5baa0af45ee as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/15d9411bb42a4fe296b5b5baa0af45ee 2024-11-14T02:45:35,457 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/15d9411bb42a4fe296b5b5baa0af45ee, entries=150, sequenceid=280, filesize=12.0 K 2024-11-14T02:45:35,457 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for 7922b12c74446660b37b836220447ee3 in 843ms, sequenceid=280, compaction requested=false 2024-11-14T02:45:35,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:35,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:35,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-14T02:45:35,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-14T02:45:35,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-14T02:45:35,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9120 sec 2024-11-14T02:45:35,462 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.9170 sec 2024-11-14T02:45:35,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:35,544 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:45:35,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7922b12c74446660b37b836220447ee3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T02:45:35,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/89459ad583e9405fb22fc0dfce098d0d is 50, key is test_row_0/A:col10/1731552335543/Put/seqid=0 2024-11-14T02:45:35,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742283_1459 (size=14741) 2024-11-14T02:45:35,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:35,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552395655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:35,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:35,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:35,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:35,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552395655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:35,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:35,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:35,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:35,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:35,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:35,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552395655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:35,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:35,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:35,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T02:45:35,693 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:35,694 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-14T02:45:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T02:45:35,695 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:35,696 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:35,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:35,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552395774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:35,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:35,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:35,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552395774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:35,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:35,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:35,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:35,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:35,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552395774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:35,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:35,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:35,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T02:45:35,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-14T02:45:35,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:35,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:35,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:35,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:35,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:35,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/89459ad583e9405fb22fc0dfce098d0d 2024-11-14T02:45:35,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/629d69be753b4dcc9af632efa1c5bd4f is 50, key is test_row_0/B:col10/1731552335543/Put/seqid=0 2024-11-14T02:45:35,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742284_1460 (size=12301) 2024-11-14T02:45:35,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552395984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:35,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:35,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552395984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:35,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:35,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:35,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:35,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:35,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:35,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552395984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:35,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-14T02:45:35,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:35,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:35,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:35,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:36,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T02:45:36,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-14T02:45:36,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:36,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:36,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:36,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:36,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:36,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:36,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:36,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552396293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:36,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:36,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552396293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:36,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:36,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552396293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:36,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-14T02:45:36,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:36,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:36,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:36,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:36,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:36,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:36,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T02:45:36,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/629d69be753b4dcc9af632efa1c5bd4f 2024-11-14T02:45:36,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/c70f1a22731944b1932fecde0600afb5 is 50, key is test_row_0/C:col10/1731552335543/Put/seqid=0 2024-11-14T02:45:36,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742285_1461 (size=12301) 2024-11-14T02:45:36,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-14T02:45:36,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:36,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:36,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:36,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:36,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:36,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-14T02:45:36,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:36,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:36,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:36,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:36,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:36,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/c70f1a22731944b1932fecde0600afb5 2024-11-14T02:45:36,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/89459ad583e9405fb22fc0dfce098d0d as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/89459ad583e9405fb22fc0dfce098d0d 2024-11-14T02:45:36,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/89459ad583e9405fb22fc0dfce098d0d, entries=200, sequenceid=294, filesize=14.4 K 2024-11-14T02:45:36,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/629d69be753b4dcc9af632efa1c5bd4f as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/629d69be753b4dcc9af632efa1c5bd4f 2024-11-14T02:45:36,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/629d69be753b4dcc9af632efa1c5bd4f, entries=150, sequenceid=294, filesize=12.0 K 2024-11-14T02:45:36,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/c70f1a22731944b1932fecde0600afb5 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/c70f1a22731944b1932fecde0600afb5 2024-11-14T02:45:36,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/c70f1a22731944b1932fecde0600afb5, entries=150, sequenceid=294, filesize=12.0 K 2024-11-14T02:45:36,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=161.02 KB/164880 for 7922b12c74446660b37b836220447ee3 in 1244ms, sequenceid=294, compaction requested=true 2024-11-14T02:45:36,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:36,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7922b12c74446660b37b836220447ee3:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:45:36,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:36,789 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:36,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7922b12c74446660b37b836220447ee3:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:45:36,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:36,789 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:36,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7922b12c74446660b37b836220447ee3:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:45:36,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:36,789 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37451 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:36,789 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:36,790 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 7922b12c74446660b37b836220447ee3/B is initiating minor compaction (all files) 2024-11-14T02:45:36,790 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 7922b12c74446660b37b836220447ee3/A is initiating minor compaction (all files) 2024-11-14T02:45:36,790 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7922b12c74446660b37b836220447ee3/B in TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:36,790 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7922b12c74446660b37b836220447ee3/A in TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:36,790 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/6a0e0e65b016487f9a3934c011ccb014, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/60fe0493daba4cc5b89a9143256afdf5, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/629d69be753b4dcc9af632efa1c5bd4f] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp, totalSize=36.6 K 2024-11-14T02:45:36,790 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/fba174fccbb2458b8119076f2af553c1, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/5026a4bf947f4e99a0ae9352d65adaca, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/89459ad583e9405fb22fc0dfce098d0d] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp, totalSize=39.0 K 2024-11-14T02:45:36,790 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting fba174fccbb2458b8119076f2af553c1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731552332686 2024-11-14T02:45:36,790 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a0e0e65b016487f9a3934c011ccb014, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731552332686 2024-11-14T02:45:36,790 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5026a4bf947f4e99a0ae9352d65adaca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731552333367 2024-11-14T02:45:36,790 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 60fe0493daba4cc5b89a9143256afdf5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731552333367 2024-11-14T02:45:36,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-14T02:45:36,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:36,791 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 7922b12c74446660b37b836220447ee3 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-14T02:45:36,792 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89459ad583e9405fb22fc0dfce098d0d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731552335527 2024-11-14T02:45:36,792 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 629d69be753b4dcc9af632efa1c5bd4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731552335527 2024-11-14T02:45:36,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/2b44a9edd2044f18a19e921d2341fb4a is 50, key is test_row_0/A:col10/1731552335632/Put/seqid=0 2024-11-14T02:45:36,804 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7922b12c74446660b37b836220447ee3#A#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:36,805 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/0f622309004c4b3fab1ea83a61ba75b7 is 50, key is test_row_0/A:col10/1731552335543/Put/seqid=0 2024-11-14T02:45:36,805 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7922b12c74446660b37b836220447ee3#B#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:36,806 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/498a66f213bf4a138fc4e5e4c9f2ea11 is 50, key is test_row_0/B:col10/1731552335543/Put/seqid=0 2024-11-14T02:45:36,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742286_1462 (size=12301) 2024-11-14T02:45:36,807 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/2b44a9edd2044f18a19e921d2341fb4a 2024-11-14T02:45:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:36,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:36,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742287_1463 (size=13051) 2024-11-14T02:45:36,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/39b8da87fb564d2986417c477f9793a8 is 50, key is test_row_0/B:col10/1731552335632/Put/seqid=0 2024-11-14T02:45:36,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742288_1464 (size=13051) 2024-11-14T02:45:36,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742289_1465 (size=12301) 2024-11-14T02:45:36,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:36,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552396821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:36,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:36,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:36,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:36,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552396822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:36,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:36,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552396824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:36,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T02:45:36,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:36,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552396933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:36,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:36,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:36,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:36,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:36,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552396933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:36,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:36,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:36,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:36,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552396934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:36,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:36,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:36,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:36,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:37,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552397143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:37,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552397143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:37,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:37,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:37,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:37,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552397143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:37,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:37,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:37,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:37,216 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/0f622309004c4b3fab1ea83a61ba75b7 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/0f622309004c4b3fab1ea83a61ba75b7 2024-11-14T02:45:37,219 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/39b8da87fb564d2986417c477f9793a8 2024-11-14T02:45:37,220 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/498a66f213bf4a138fc4e5e4c9f2ea11 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/498a66f213bf4a138fc4e5e4c9f2ea11 2024-11-14T02:45:37,220 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7922b12c74446660b37b836220447ee3/A of 7922b12c74446660b37b836220447ee3 into 0f622309004c4b3fab1ea83a61ba75b7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:37,220 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:37,220 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., storeName=7922b12c74446660b37b836220447ee3/A, priority=13, startTime=1731552336789; duration=0sec 2024-11-14T02:45:37,221 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:37,221 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7922b12c74446660b37b836220447ee3:A 2024-11-14T02:45:37,221 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:37,221 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37451 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:37,221 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 7922b12c74446660b37b836220447ee3/C is initiating minor compaction (all files) 2024-11-14T02:45:37,221 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7922b12c74446660b37b836220447ee3/C in TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:37,222 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/345a6d6a69d843efb62604c4596a7aea, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/15d9411bb42a4fe296b5b5baa0af45ee, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/c70f1a22731944b1932fecde0600afb5] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp, totalSize=36.6 K 2024-11-14T02:45:37,222 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 345a6d6a69d843efb62604c4596a7aea, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731552332686 2024-11-14T02:45:37,223 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15d9411bb42a4fe296b5b5baa0af45ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731552333367 2024-11-14T02:45:37,224 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting c70f1a22731944b1932fecde0600afb5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731552335527 2024-11-14T02:45:37,225 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7922b12c74446660b37b836220447ee3/B of 7922b12c74446660b37b836220447ee3 into 498a66f213bf4a138fc4e5e4c9f2ea11(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:37,225 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:37,225 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., storeName=7922b12c74446660b37b836220447ee3/B, priority=13, startTime=1731552336789; duration=0sec 2024-11-14T02:45:37,225 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:37,225 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7922b12c74446660b37b836220447ee3:B 2024-11-14T02:45:37,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/ed89b8c645d147c48f0da8e67ba22eaf is 50, key is test_row_0/C:col10/1731552335632/Put/seqid=0 2024-11-14T02:45:37,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742290_1466 (size=12301) 2024-11-14T02:45:37,233 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7922b12c74446660b37b836220447ee3#C#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:37,233 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/34cb2fb40edf4a47a3530cdb721de5b4 is 50, key is test_row_0/C:col10/1731552335543/Put/seqid=0 2024-11-14T02:45:37,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742291_1467 (size=13051) 2024-11-14T02:45:37,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:37,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552397453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:37,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:37,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552397453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:37,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:37,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:37,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:37,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:37,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:37,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:37,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:37,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552397463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:37,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:37,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:37,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:37,629 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/ed89b8c645d147c48f0da8e67ba22eaf 2024-11-14T02:45:37,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/2b44a9edd2044f18a19e921d2341fb4a as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/2b44a9edd2044f18a19e921d2341fb4a 2024-11-14T02:45:37,636 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/2b44a9edd2044f18a19e921d2341fb4a, entries=150, sequenceid=321, filesize=12.0 K 2024-11-14T02:45:37,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/39b8da87fb564d2986417c477f9793a8 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/39b8da87fb564d2986417c477f9793a8 2024-11-14T02:45:37,639 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/34cb2fb40edf4a47a3530cdb721de5b4 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/34cb2fb40edf4a47a3530cdb721de5b4 2024-11-14T02:45:37,639 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/39b8da87fb564d2986417c477f9793a8, entries=150, sequenceid=321, filesize=12.0 K 2024-11-14T02:45:37,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/ed89b8c645d147c48f0da8e67ba22eaf as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ed89b8c645d147c48f0da8e67ba22eaf 2024-11-14T02:45:37,642 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ed89b8c645d147c48f0da8e67ba22eaf, entries=150, sequenceid=321, filesize=12.0 K 2024-11-14T02:45:37,642 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7922b12c74446660b37b836220447ee3/C of 7922b12c74446660b37b836220447ee3 into 34cb2fb40edf4a47a3530cdb721de5b4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:37,642 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:37,642 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., storeName=7922b12c74446660b37b836220447ee3/C, priority=13, startTime=1731552336789; duration=0sec 2024-11-14T02:45:37,642 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:37,642 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7922b12c74446660b37b836220447ee3:C 2024-11-14T02:45:37,643 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 7922b12c74446660b37b836220447ee3 in 852ms, sequenceid=321, compaction requested=false 2024-11-14T02:45:37,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:37,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:37,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-14T02:45:37,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-14T02:45:37,646 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-14T02:45:37,646 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9480 sec 2024-11-14T02:45:37,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.9520 sec 2024-11-14T02:45:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T02:45:37,843 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:37,844 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:37,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-14T02:45:37,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T02:45:37,845 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:37,846 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:37,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T02:45:37,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:37,967 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:45:37,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7922b12c74446660b37b836220447ee3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T02:45:37,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/f4685145dae242199f0b926da3c82bb2 is 50, key is test_row_0/A:col10/1731552337965/Put/seqid=0 2024-11-14T02:45:37,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742292_1468 (size=14741) 2024-11-14T02:45:37,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/f4685145dae242199f0b926da3c82bb2 2024-11-14T02:45:37,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/42a04109630e425792828abfc3d299da is 50, key is test_row_0/B:col10/1731552337965/Put/seqid=0 2024-11-14T02:45:37,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742293_1469 (size=12301) 2024-11-14T02:45:37,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T02:45:37,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:37,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:37,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:37,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:37,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552398028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:38,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:38,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552398028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:38,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:38,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552398029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:38,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:38,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552398134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:38,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:38,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:38,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552398143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:38,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:38,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:38,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552398143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:38,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:38,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T02:45:38,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:38,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:38,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:38,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T02:45:38,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T02:45:38,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:38,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:38,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:38,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:38,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552398343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:38,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:38,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:38,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552398354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:38,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:38,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:38,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552398354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:38,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:38,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/42a04109630e425792828abfc3d299da 2024-11-14T02:45:38,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/25af56cdf6864512a9a3a9ecb77293aa is 50, key is test_row_0/C:col10/1731552337965/Put/seqid=0 2024-11-14T02:45:38,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742294_1470 (size=12301) 2024-11-14T02:45:38,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T02:45:38,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:38,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:38,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:38,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T02:45:38,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T02:45:38,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:38,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:38,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:38,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552398653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:38,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:38,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552398663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:38,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552398663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:38,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:38,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T02:45:38,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:38,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:38,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:38,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/25af56cdf6864512a9a3a9ecb77293aa 2024-11-14T02:45:38,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/f4685145dae242199f0b926da3c82bb2 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/f4685145dae242199f0b926da3c82bb2 2024-11-14T02:45:38,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/f4685145dae242199f0b926da3c82bb2, entries=200, sequenceid=335, filesize=14.4 K 2024-11-14T02:45:38,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/42a04109630e425792828abfc3d299da as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/42a04109630e425792828abfc3d299da 2024-11-14T02:45:38,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/42a04109630e425792828abfc3d299da, entries=150, sequenceid=335, filesize=12.0 K 2024-11-14T02:45:38,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/25af56cdf6864512a9a3a9ecb77293aa as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/25af56cdf6864512a9a3a9ecb77293aa 2024-11-14T02:45:38,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/25af56cdf6864512a9a3a9ecb77293aa, entries=150, sequenceid=335, filesize=12.0 K 2024-11-14T02:45:38,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 7922b12c74446660b37b836220447ee3 in 837ms, sequenceid=335, compaction requested=true 2024-11-14T02:45:38,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:38,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7922b12c74446660b37b836220447ee3:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:45:38,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:38,805 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:38,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7922b12c74446660b37b836220447ee3:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:45:38,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:38,805 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:38,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7922b12c74446660b37b836220447ee3:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:45:38,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:38,805 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:38,805 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:38,805 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 7922b12c74446660b37b836220447ee3/B is initiating minor compaction (all files) 2024-11-14T02:45:38,805 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 7922b12c74446660b37b836220447ee3/A is initiating minor compaction (all files) 2024-11-14T02:45:38,806 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7922b12c74446660b37b836220447ee3/A in TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:38,806 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7922b12c74446660b37b836220447ee3/B in TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:38,806 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/498a66f213bf4a138fc4e5e4c9f2ea11, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/39b8da87fb564d2986417c477f9793a8, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/42a04109630e425792828abfc3d299da] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp, totalSize=36.8 K 2024-11-14T02:45:38,806 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/0f622309004c4b3fab1ea83a61ba75b7, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/2b44a9edd2044f18a19e921d2341fb4a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/f4685145dae242199f0b926da3c82bb2] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp, totalSize=39.2 K 2024-11-14T02:45:38,806 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 498a66f213bf4a138fc4e5e4c9f2ea11, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731552335527 2024-11-14T02:45:38,806 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f622309004c4b3fab1ea83a61ba75b7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731552335527 2024-11-14T02:45:38,806 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b44a9edd2044f18a19e921d2341fb4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731552335610 2024-11-14T02:45:38,806 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 39b8da87fb564d2986417c477f9793a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731552335610 2024-11-14T02:45:38,806 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4685145dae242199f0b926da3c82bb2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731552336821 2024-11-14T02:45:38,806 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 42a04109630e425792828abfc3d299da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731552336823 2024-11-14T02:45:38,812 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7922b12c74446660b37b836220447ee3#B#compaction#401 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:38,812 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7922b12c74446660b37b836220447ee3#A#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:38,812 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/b6f5bcf41f164b54b389856b9c5bd540 is 50, key is test_row_0/B:col10/1731552337965/Put/seqid=0 2024-11-14T02:45:38,812 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/5825cd99fc884c1393a8eedc0a48d8c8 is 50, key is test_row_0/A:col10/1731552337965/Put/seqid=0 2024-11-14T02:45:38,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742295_1471 (size=13153) 2024-11-14T02:45:38,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742296_1472 (size=13153) 2024-11-14T02:45:38,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T02:45:38,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:38,914 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 7922b12c74446660b37b836220447ee3 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T02:45:38,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/65e98c04ff4849729969ea7de098f4e0 is 50, key is test_row_0/A:col10/1731552338018/Put/seqid=0 2024-11-14T02:45:38,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742297_1473 (size=12301) 2024-11-14T02:45:38,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T02:45:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:39,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. as already flushing 2024-11-14T02:45:39,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:39,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552399191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:39,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:39,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:39,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552399192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:39,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:39,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:39,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552399195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:39,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:39,223 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/b6f5bcf41f164b54b389856b9c5bd540 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/b6f5bcf41f164b54b389856b9c5bd540 2024-11-14T02:45:39,226 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7922b12c74446660b37b836220447ee3/B of 7922b12c74446660b37b836220447ee3 into b6f5bcf41f164b54b389856b9c5bd540(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:39,226 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:39,226 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., storeName=7922b12c74446660b37b836220447ee3/B, priority=13, startTime=1731552338805; duration=0sec 2024-11-14T02:45:39,226 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:39,226 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7922b12c74446660b37b836220447ee3:B 2024-11-14T02:45:39,226 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:39,227 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:39,227 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 7922b12c74446660b37b836220447ee3/C is initiating minor compaction (all files) 2024-11-14T02:45:39,227 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7922b12c74446660b37b836220447ee3/C in TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:39,227 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/34cb2fb40edf4a47a3530cdb721de5b4, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ed89b8c645d147c48f0da8e67ba22eaf, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/25af56cdf6864512a9a3a9ecb77293aa] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp, totalSize=36.8 K 2024-11-14T02:45:39,228 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 34cb2fb40edf4a47a3530cdb721de5b4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731552335527 2024-11-14T02:45:39,228 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting ed89b8c645d147c48f0da8e67ba22eaf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731552335610 2024-11-14T02:45:39,229 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 25af56cdf6864512a9a3a9ecb77293aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731552336823 2024-11-14T02:45:39,229 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/5825cd99fc884c1393a8eedc0a48d8c8 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/5825cd99fc884c1393a8eedc0a48d8c8 2024-11-14T02:45:39,233 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7922b12c74446660b37b836220447ee3/A of 7922b12c74446660b37b836220447ee3 into 5825cd99fc884c1393a8eedc0a48d8c8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:39,233 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:39,233 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., storeName=7922b12c74446660b37b836220447ee3/A, priority=13, startTime=1731552338805; duration=0sec 2024-11-14T02:45:39,233 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:39,233 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7922b12c74446660b37b836220447ee3:A 2024-11-14T02:45:39,234 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7922b12c74446660b37b836220447ee3#C#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:39,235 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/1ae11521914843de918b6da83a5840f3 is 50, key is test_row_0/C:col10/1731552337965/Put/seqid=0 2024-11-14T02:45:39,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742298_1474 (size=13153) 2024-11-14T02:45:39,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552399303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:39,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:39,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552399304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:39,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:39,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552399314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:39,321 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/65e98c04ff4849729969ea7de098f4e0 2024-11-14T02:45:39,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/7d101c15d9c44522af868edeffff9d65 is 50, key is test_row_0/B:col10/1731552338018/Put/seqid=0 2024-11-14T02:45:39,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742299_1475 (size=12301) 2024-11-14T02:45:39,330 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/7d101c15d9c44522af868edeffff9d65 2024-11-14T02:45:39,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/929f51a56e3e4d2d9506de5830af8539 is 50, key is test_row_0/C:col10/1731552338018/Put/seqid=0 2024-11-14T02:45:39,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742300_1476 (size=12301) 2024-11-14T02:45:39,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:39,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731552399513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:39,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:39,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:39,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731552399513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:39,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:39,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:39,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1731552399524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:39,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:39,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731552399553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:39,561 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_2' in 7922b12c74446660b37b836220447ee3 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18286 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:45:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:39,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:39,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731552399613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:39,619 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_2' in 7922b12c74446660b37b836220447ee3 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18343 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:45:39,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:39,642 INFO [Thread-1810 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1810. 2024-11-14T02:45:39,642 DEBUG [Thread-1810 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:39,642 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/1ae11521914843de918b6da83a5840f3 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/1ae11521914843de918b6da83a5840f3 2024-11-14T02:45:39,642 DEBUG [Thread-1810 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:39,642 DEBUG [Thread-1810 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:39,642 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:39,646 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7922b12c74446660b37b836220447ee3/C of 7922b12c74446660b37b836220447ee3 into 1ae11521914843de918b6da83a5840f3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:39,646 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:39,646 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., storeName=7922b12c74446660b37b836220447ee3/C, priority=13, startTime=1731552338805; duration=0sec 2024-11-14T02:45:39,646 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:39,646 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7922b12c74446660b37b836220447ee3:C 2024-11-14T02:45:39,647 INFO [Thread-1816 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1816. 2024-11-14T02:45:39,647 DEBUG [Thread-1816 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:39,647 DEBUG [Thread-1816 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:39,647 DEBUG [Thread-1816 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:39,647 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:39,648 INFO [Thread-1814 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1814. 2024-11-14T02:45:39,648 DEBUG [Thread-1814 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:39,648 DEBUG [Thread-1814 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:39,648 DEBUG [Thread-1814 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:39,648 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:39,650 INFO [Thread-1808 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1808. 2024-11-14T02:45:39,651 DEBUG [Thread-1808 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:39,651 DEBUG [Thread-1808 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:39,651 DEBUG [Thread-1808 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:39,651 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:39,654 INFO [Thread-1812 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1812. 2024-11-14T02:45:39,654 DEBUG [Thread-1812 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:39,654 DEBUG [Thread-1812 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:39,654 DEBUG [Thread-1812 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:39,654 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:39,739 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/929f51a56e3e4d2d9506de5830af8539 2024-11-14T02:45:39,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/65e98c04ff4849729969ea7de098f4e0 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/65e98c04ff4849729969ea7de098f4e0 2024-11-14T02:45:39,750 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T02:45:39,753 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/65e98c04ff4849729969ea7de098f4e0, entries=150, sequenceid=360, filesize=12.0 K 2024-11-14T02:45:39,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/7d101c15d9c44522af868edeffff9d65 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/7d101c15d9c44522af868edeffff9d65 2024-11-14T02:45:39,759 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/7d101c15d9c44522af868edeffff9d65, entries=150, sequenceid=360, filesize=12.0 K 2024-11-14T02:45:39,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/929f51a56e3e4d2d9506de5830af8539 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/929f51a56e3e4d2d9506de5830af8539 2024-11-14T02:45:39,764 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/929f51a56e3e4d2d9506de5830af8539, entries=150, sequenceid=360, filesize=12.0 K 2024-11-14T02:45:39,765 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 7922b12c74446660b37b836220447ee3 in 851ms, sequenceid=360, compaction requested=false 2024-11-14T02:45:39,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:39,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:39,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-14T02:45:39,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-14T02:45:39,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-14T02:45:39,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9210 sec 2024-11-14T02:45:39,771 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.9260 sec 2024-11-14T02:45:39,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:39,827 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:45:39,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7922b12c74446660b37b836220447ee3 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T02:45:39,827 INFO [Thread-1801 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1801. 2024-11-14T02:45:39,827 DEBUG [Thread-1801 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:39,828 DEBUG [Thread-1801 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:39,828 DEBUG [Thread-1801 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:39,828 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:39,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/5b344e348b9344c79ff69d00d3f3eca0 is 50, key is test_row_0/A:col10/1731552339194/Put/seqid=0 2024-11-14T02:45:39,836 INFO [Thread-1805 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1805. 2024-11-14T02:45:39,836 DEBUG [Thread-1805 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:39,836 DEBUG [Thread-1805 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:39,837 DEBUG [Thread-1805 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:39,837 INFO [Thread-1797 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1797. 2024-11-14T02:45:39,837 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:39,837 DEBUG [Thread-1797 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:39,837 DEBUG [Thread-1797 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:39,837 DEBUG [Thread-1797 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:39,837 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:39,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742301_1477 (size=12301) 2024-11-14T02:45:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T02:45:40,005 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:40,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/5b344e348b9344c79ff69d00d3f3eca0 2024-11-14T02:45:40,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/1767ec8de9a24aa2bc15c7fcaba8a55b is 50, key is test_row_0/B:col10/1731552339194/Put/seqid=0 2024-11-14T02:45:40,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742302_1478 (size=12301) 2024-11-14T02:45:40,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/1767ec8de9a24aa2bc15c7fcaba8a55b 2024-11-14T02:45:40,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/ebefb91de92b4347bc367eab9d57cc0a is 50, key is test_row_0/C:col10/1731552339194/Put/seqid=0 2024-11-14T02:45:40,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742303_1479 (size=12301) 2024-11-14T02:45:41,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/ebefb91de92b4347bc367eab9d57cc0a 2024-11-14T02:45:41,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/5b344e348b9344c79ff69d00d3f3eca0 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/5b344e348b9344c79ff69d00d3f3eca0 2024-11-14T02:45:41,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/5b344e348b9344c79ff69d00d3f3eca0, entries=150, sequenceid=375, filesize=12.0 K 2024-11-14T02:45:41,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/1767ec8de9a24aa2bc15c7fcaba8a55b as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/1767ec8de9a24aa2bc15c7fcaba8a55b 2024-11-14T02:45:41,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/1767ec8de9a24aa2bc15c7fcaba8a55b, entries=150, sequenceid=375, filesize=12.0 K 2024-11-14T02:45:41,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/ebefb91de92b4347bc367eab9d57cc0a as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ebefb91de92b4347bc367eab9d57cc0a 2024-11-14T02:45:41,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ebefb91de92b4347bc367eab9d57cc0a, entries=150, sequenceid=375, filesize=12.0 K 2024-11-14T02:45:41,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=13.42 KB/13740 for 7922b12c74446660b37b836220447ee3 in 1275ms, sequenceid=375, compaction requested=true 2024-11-14T02:45:41,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:41,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7922b12c74446660b37b836220447ee3:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:45:41,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:41,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7922b12c74446660b37b836220447ee3:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:45:41,102 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:41,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:41,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7922b12c74446660b37b836220447ee3:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:45:41,102 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:41,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:41,103 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:41,103 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:41,103 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): 7922b12c74446660b37b836220447ee3/A is initiating minor compaction (all files) 2024-11-14T02:45:41,103 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 7922b12c74446660b37b836220447ee3/B is initiating minor compaction (all files) 2024-11-14T02:45:41,103 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7922b12c74446660b37b836220447ee3/B in TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:41,103 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7922b12c74446660b37b836220447ee3/A in TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:41,103 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/5825cd99fc884c1393a8eedc0a48d8c8, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/65e98c04ff4849729969ea7de098f4e0, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/5b344e348b9344c79ff69d00d3f3eca0] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp, totalSize=36.9 K 2024-11-14T02:45:41,103 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/b6f5bcf41f164b54b389856b9c5bd540, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/7d101c15d9c44522af868edeffff9d65, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/1767ec8de9a24aa2bc15c7fcaba8a55b] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp, totalSize=36.9 K 2024-11-14T02:45:41,104 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5825cd99fc884c1393a8eedc0a48d8c8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731552336823 2024-11-14T02:45:41,104 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting b6f5bcf41f164b54b389856b9c5bd540, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731552336823 2024-11-14T02:45:41,104 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d101c15d9c44522af868edeffff9d65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731552338018 2024-11-14T02:45:41,104 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65e98c04ff4849729969ea7de098f4e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731552338018 2024-11-14T02:45:41,104 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b344e348b9344c79ff69d00d3f3eca0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731552339191 2024-11-14T02:45:41,104 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 1767ec8de9a24aa2bc15c7fcaba8a55b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731552339191 2024-11-14T02:45:41,112 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7922b12c74446660b37b836220447ee3#A#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:41,112 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7922b12c74446660b37b836220447ee3#B#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:41,113 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/1a6e6eace70b42cdb0195c65ef13cd00 is 50, key is test_row_0/A:col10/1731552339194/Put/seqid=0 2024-11-14T02:45:41,113 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/f7934d3bab3c4a42a5a66412de6d8b18 is 50, key is test_row_0/B:col10/1731552339194/Put/seqid=0 2024-11-14T02:45:41,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742305_1481 (size=13255) 2024-11-14T02:45:41,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742304_1480 (size=13255) 2024-11-14T02:45:41,528 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/f7934d3bab3c4a42a5a66412de6d8b18 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/f7934d3bab3c4a42a5a66412de6d8b18 2024-11-14T02:45:41,529 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/1a6e6eace70b42cdb0195c65ef13cd00 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/1a6e6eace70b42cdb0195c65ef13cd00 2024-11-14T02:45:41,533 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7922b12c74446660b37b836220447ee3/B of 7922b12c74446660b37b836220447ee3 into f7934d3bab3c4a42a5a66412de6d8b18(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:41,533 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7922b12c74446660b37b836220447ee3/A of 7922b12c74446660b37b836220447ee3 into 1a6e6eace70b42cdb0195c65ef13cd00(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:41,533 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:41,533 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:41,533 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., storeName=7922b12c74446660b37b836220447ee3/A, priority=13, startTime=1731552341102; duration=0sec 2024-11-14T02:45:41,533 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., storeName=7922b12c74446660b37b836220447ee3/B, priority=13, startTime=1731552341102; duration=0sec 2024-11-14T02:45:41,533 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:41,533 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7922b12c74446660b37b836220447ee3:B 2024-11-14T02:45:41,533 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:41,533 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7922b12c74446660b37b836220447ee3:A 2024-11-14T02:45:41,533 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:41,534 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:41,534 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): 7922b12c74446660b37b836220447ee3/C is initiating minor compaction (all files) 2024-11-14T02:45:41,534 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7922b12c74446660b37b836220447ee3/C in TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:41,534 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/1ae11521914843de918b6da83a5840f3, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/929f51a56e3e4d2d9506de5830af8539, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ebefb91de92b4347bc367eab9d57cc0a] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp, totalSize=36.9 K 2024-11-14T02:45:41,534 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ae11521914843de918b6da83a5840f3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731552336823 2024-11-14T02:45:41,535 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 929f51a56e3e4d2d9506de5830af8539, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731552338018 2024-11-14T02:45:41,535 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting ebefb91de92b4347bc367eab9d57cc0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731552339191 2024-11-14T02:45:41,540 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7922b12c74446660b37b836220447ee3#C#compaction#412 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:41,541 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/876197c18a25414fb4cc21da59c9ce03 is 50, key is test_row_0/C:col10/1731552339194/Put/seqid=0 2024-11-14T02:45:41,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742306_1482 (size=13255) 2024-11-14T02:45:41,957 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/876197c18a25414fb4cc21da59c9ce03 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/876197c18a25414fb4cc21da59c9ce03 2024-11-14T02:45:41,962 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7922b12c74446660b37b836220447ee3/C of 7922b12c74446660b37b836220447ee3 into 876197c18a25414fb4cc21da59c9ce03(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:41,962 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7922b12c74446660b37b836220447ee3: 2024-11-14T02:45:41,962 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., storeName=7922b12c74446660b37b836220447ee3/C, priority=13, startTime=1731552341102; duration=0sec 2024-11-14T02:45:41,963 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:41,963 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7922b12c74446660b37b836220447ee3:C 2024-11-14T02:45:49,658 INFO [Thread-1799 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1799. 2024-11-14T02:45:49,658 DEBUG [Thread-1799 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:49,658 DEBUG [Thread-1799 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:49,658 DEBUG [Thread-1799 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:49,658 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:49,708 INFO [Thread-1803 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1803. 2024-11-14T02:45:49,708 DEBUG [Thread-1803 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:49,708 DEBUG [Thread-1803 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:49,708 DEBUG [Thread-1803 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:49,709 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:49,709 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T02:45:49,709 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-14T02:45:49,709 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-14T02:45:49,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-14T02:45:49,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-14T02:45:49,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-11-14T02:45:49,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T02:45:49,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T02:45:49,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1578 2024-11-14T02:45:49,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4733 rows 2024-11-14T02:45:49,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1568 2024-11-14T02:45:49,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4698 rows 2024-11-14T02:45:49,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1568 2024-11-14T02:45:49,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4700 rows 2024-11-14T02:45:49,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1575 2024-11-14T02:45:49,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4725 rows 2024-11-14T02:45:49,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1580 2024-11-14T02:45:49,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4739 rows 2024-11-14T02:45:49,710 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T02:45:49,710 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T02:45:49,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:49,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:49,711 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:49,712 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T02:45:49,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T02:45:49,718 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552349718"}]},"ts":"1731552349718"} 2024-11-14T02:45:49,720 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T02:45:49,720 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T02:45:49,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T02:45:49,722 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7922b12c74446660b37b836220447ee3, UNASSIGN}] 2024-11-14T02:45:49,723 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7922b12c74446660b37b836220447ee3, UNASSIGN 2024-11-14T02:45:49,723 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=7922b12c74446660b37b836220447ee3, regionState=CLOSING, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:49,724 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7922b12c74446660b37b836220447ee3, UNASSIGN because future has completed 2024-11-14T02:45:49,725 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T02:45:49,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259}] 2024-11-14T02:45:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T02:45:49,878 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:49,878 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T02:45:49,879 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing 7922b12c74446660b37b836220447ee3, disabling compactions & flushes 2024-11-14T02:45:49,879 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:49,879 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:49,879 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. after waiting 0 ms 2024-11-14T02:45:49,879 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:49,879 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(2902): Flushing 7922b12c74446660b37b836220447ee3 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-14T02:45:49,889 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/f3dc6d3ace314a1ca1b9d9af07a170e2 is 50, key is test_row_0/A:col10/1731552339834/Put/seqid=0 2024-11-14T02:45:49,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742307_1483 (size=9857) 2024-11-14T02:45:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T02:45:50,295 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/f3dc6d3ace314a1ca1b9d9af07a170e2 2024-11-14T02:45:50,309 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/58738d8fae2f42d2a3dd7b58c80f6df6 is 50, key is test_row_0/B:col10/1731552339834/Put/seqid=0 2024-11-14T02:45:50,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742308_1484 (size=9857) 2024-11-14T02:45:50,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T02:45:50,714 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/58738d8fae2f42d2a3dd7b58c80f6df6 2024-11-14T02:45:50,726 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/bd427f78e71444aca6430801ba454b64 is 50, key is test_row_0/C:col10/1731552339834/Put/seqid=0 2024-11-14T02:45:50,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742309_1485 (size=9857) 2024-11-14T02:45:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T02:45:51,131 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/bd427f78e71444aca6430801ba454b64 2024-11-14T02:45:51,140 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/A/f3dc6d3ace314a1ca1b9d9af07a170e2 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/f3dc6d3ace314a1ca1b9d9af07a170e2 2024-11-14T02:45:51,144 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/f3dc6d3ace314a1ca1b9d9af07a170e2, entries=100, sequenceid=385, filesize=9.6 K 2024-11-14T02:45:51,144 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/B/58738d8fae2f42d2a3dd7b58c80f6df6 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/58738d8fae2f42d2a3dd7b58c80f6df6 2024-11-14T02:45:51,148 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/58738d8fae2f42d2a3dd7b58c80f6df6, entries=100, sequenceid=385, filesize=9.6 K 2024-11-14T02:45:51,149 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/.tmp/C/bd427f78e71444aca6430801ba454b64 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/bd427f78e71444aca6430801ba454b64 2024-11-14T02:45:51,151 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/bd427f78e71444aca6430801ba454b64, entries=100, sequenceid=385, filesize=9.6 K 2024-11-14T02:45:51,152 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 7922b12c74446660b37b836220447ee3 in 1273ms, sequenceid=385, compaction requested=false 2024-11-14T02:45:51,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/25a90819454c44afb062d2738ec3a04a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/eccde65102a74122a7a38168b6f64729, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/b2e19918caae4cf4ab76a6a02e52e398, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/24145443f4974e5ba44f3374f7f33033, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/917e6ed4160740ca90b9348c129bac42, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/0c3b9932e8b046d6bb04500ffe8dcc79, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/a00c545f794c42a78dcf63c5cee75be0, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/039c350d92ee47129073a44e2775ec61, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/06b8efaf8638499cbfb6168d9f88125c, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/ac73b53cdefe428e9c8c33735b77e7d7, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/f5a3fb56526743c7ae67d02fa868c6e0, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/ef7e73d870694749849a813ee5a97e46, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/a11017f1f41949bda750ad26a042579d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/06fdae0922774dea82ba774cf5c25be9, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/85cb1ca169b74657aea51ed2199ca808, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/287f75944f4e4def87f766f57f300a74, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/0f5b9ff81f5e42b49789a08e2be1ec38, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/2bf5ee6e7a484ee2988533b676a1d377, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/afaafc734ce14b7bae02f78bba12e2b8, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/fba174fccbb2458b8119076f2af553c1, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/5026a4bf947f4e99a0ae9352d65adaca, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/89459ad583e9405fb22fc0dfce098d0d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/0f622309004c4b3fab1ea83a61ba75b7, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/2b44a9edd2044f18a19e921d2341fb4a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/f4685145dae242199f0b926da3c82bb2, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/5825cd99fc884c1393a8eedc0a48d8c8, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/65e98c04ff4849729969ea7de098f4e0, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/5b344e348b9344c79ff69d00d3f3eca0] to archive 2024-11-14T02:45:51,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:45:51,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/25a90819454c44afb062d2738ec3a04a to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/25a90819454c44afb062d2738ec3a04a 2024-11-14T02:45:51,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/eccde65102a74122a7a38168b6f64729 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/eccde65102a74122a7a38168b6f64729 2024-11-14T02:45:51,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/b2e19918caae4cf4ab76a6a02e52e398 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/b2e19918caae4cf4ab76a6a02e52e398 2024-11-14T02:45:51,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/24145443f4974e5ba44f3374f7f33033 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/24145443f4974e5ba44f3374f7f33033 2024-11-14T02:45:51,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/917e6ed4160740ca90b9348c129bac42 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/917e6ed4160740ca90b9348c129bac42 2024-11-14T02:45:51,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/0c3b9932e8b046d6bb04500ffe8dcc79 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/0c3b9932e8b046d6bb04500ffe8dcc79 2024-11-14T02:45:51,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/a00c545f794c42a78dcf63c5cee75be0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/a00c545f794c42a78dcf63c5cee75be0 2024-11-14T02:45:51,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/039c350d92ee47129073a44e2775ec61 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/039c350d92ee47129073a44e2775ec61 2024-11-14T02:45:51,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/06b8efaf8638499cbfb6168d9f88125c to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/06b8efaf8638499cbfb6168d9f88125c 2024-11-14T02:45:51,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/ac73b53cdefe428e9c8c33735b77e7d7 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/ac73b53cdefe428e9c8c33735b77e7d7 2024-11-14T02:45:51,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/f5a3fb56526743c7ae67d02fa868c6e0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/f5a3fb56526743c7ae67d02fa868c6e0 2024-11-14T02:45:51,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/ef7e73d870694749849a813ee5a97e46 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/ef7e73d870694749849a813ee5a97e46 2024-11-14T02:45:51,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/a11017f1f41949bda750ad26a042579d to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/a11017f1f41949bda750ad26a042579d 2024-11-14T02:45:51,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/06fdae0922774dea82ba774cf5c25be9 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/06fdae0922774dea82ba774cf5c25be9 2024-11-14T02:45:51,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/85cb1ca169b74657aea51ed2199ca808 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/85cb1ca169b74657aea51ed2199ca808 2024-11-14T02:45:51,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/287f75944f4e4def87f766f57f300a74 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/287f75944f4e4def87f766f57f300a74 2024-11-14T02:45:51,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/0f5b9ff81f5e42b49789a08e2be1ec38 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/0f5b9ff81f5e42b49789a08e2be1ec38 2024-11-14T02:45:51,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/2bf5ee6e7a484ee2988533b676a1d377 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/2bf5ee6e7a484ee2988533b676a1d377 2024-11-14T02:45:51,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/afaafc734ce14b7bae02f78bba12e2b8 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/afaafc734ce14b7bae02f78bba12e2b8 2024-11-14T02:45:51,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/fba174fccbb2458b8119076f2af553c1 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/fba174fccbb2458b8119076f2af553c1 2024-11-14T02:45:51,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/5026a4bf947f4e99a0ae9352d65adaca to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/5026a4bf947f4e99a0ae9352d65adaca 2024-11-14T02:45:51,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/89459ad583e9405fb22fc0dfce098d0d to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/89459ad583e9405fb22fc0dfce098d0d 2024-11-14T02:45:51,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/0f622309004c4b3fab1ea83a61ba75b7 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/0f622309004c4b3fab1ea83a61ba75b7 2024-11-14T02:45:51,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/2b44a9edd2044f18a19e921d2341fb4a to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/2b44a9edd2044f18a19e921d2341fb4a 2024-11-14T02:45:51,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/f4685145dae242199f0b926da3c82bb2 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/f4685145dae242199f0b926da3c82bb2 2024-11-14T02:45:51,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/5825cd99fc884c1393a8eedc0a48d8c8 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/5825cd99fc884c1393a8eedc0a48d8c8 2024-11-14T02:45:51,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/65e98c04ff4849729969ea7de098f4e0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/65e98c04ff4849729969ea7de098f4e0 2024-11-14T02:45:51,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/5b344e348b9344c79ff69d00d3f3eca0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/5b344e348b9344c79ff69d00d3f3eca0 2024-11-14T02:45:51,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/dfb2e52fed0d470eaf67be93fa68047a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/848fcd76635c4d9a84e5095d9dbbb838, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/863ddf5d964c48f09de888aba8e4331c, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/67248d8af43b496d831ea69d08512a84, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/eb5e200812d04692a329768018c4036b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/49637c0dde374657b45247e70da05c95, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/6819268912674f5eaaae72d5acee1860, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/6dc2565424b24fc09cebd5e2ffc33404, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/e8d2c248b05d4db18b09eca7c4c267db, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/305518e222674ce98b8d999d5671cfe1, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/c4513b5b81bc40c8969f67fb1f8f03e8, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/7b03d8dbeeb54634a9cfd20f9028694f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/edf2de2feea147df8e9207fa35a57bcc, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/8dc5226adb46433b857ad2c44bd3eb04, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/e59628e58fe54929b9703bffd2641cfb, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/8e6791f9902b4e518dd573e72cba9431, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/667fc89c86a845bc91c2671718bae26c, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/d9300bee19b04daabfaa8b463fd2edc0, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/6a0e0e65b016487f9a3934c011ccb014, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/4c1a551b4aea4883b7beb1126acd26bd, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/60fe0493daba4cc5b89a9143256afdf5, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/498a66f213bf4a138fc4e5e4c9f2ea11, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/629d69be753b4dcc9af632efa1c5bd4f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/39b8da87fb564d2986417c477f9793a8, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/b6f5bcf41f164b54b389856b9c5bd540, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/42a04109630e425792828abfc3d299da, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/7d101c15d9c44522af868edeffff9d65, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/1767ec8de9a24aa2bc15c7fcaba8a55b] to archive 2024-11-14T02:45:51,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:45:51,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/dfb2e52fed0d470eaf67be93fa68047a to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/dfb2e52fed0d470eaf67be93fa68047a 2024-11-14T02:45:51,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/848fcd76635c4d9a84e5095d9dbbb838 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/848fcd76635c4d9a84e5095d9dbbb838 2024-11-14T02:45:51,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/863ddf5d964c48f09de888aba8e4331c to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/863ddf5d964c48f09de888aba8e4331c 2024-11-14T02:45:51,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/67248d8af43b496d831ea69d08512a84 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/67248d8af43b496d831ea69d08512a84 2024-11-14T02:45:51,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/eb5e200812d04692a329768018c4036b to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/eb5e200812d04692a329768018c4036b 2024-11-14T02:45:51,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/49637c0dde374657b45247e70da05c95 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/49637c0dde374657b45247e70da05c95 2024-11-14T02:45:51,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/6819268912674f5eaaae72d5acee1860 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/6819268912674f5eaaae72d5acee1860 2024-11-14T02:45:51,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/6dc2565424b24fc09cebd5e2ffc33404 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/6dc2565424b24fc09cebd5e2ffc33404 2024-11-14T02:45:51,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/e8d2c248b05d4db18b09eca7c4c267db to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/e8d2c248b05d4db18b09eca7c4c267db 2024-11-14T02:45:51,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/305518e222674ce98b8d999d5671cfe1 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/305518e222674ce98b8d999d5671cfe1 2024-11-14T02:45:51,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/c4513b5b81bc40c8969f67fb1f8f03e8 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/c4513b5b81bc40c8969f67fb1f8f03e8 2024-11-14T02:45:51,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/7b03d8dbeeb54634a9cfd20f9028694f to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/7b03d8dbeeb54634a9cfd20f9028694f 2024-11-14T02:45:51,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/edf2de2feea147df8e9207fa35a57bcc to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/edf2de2feea147df8e9207fa35a57bcc 2024-11-14T02:45:51,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/8dc5226adb46433b857ad2c44bd3eb04 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/8dc5226adb46433b857ad2c44bd3eb04 2024-11-14T02:45:51,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/e59628e58fe54929b9703bffd2641cfb to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/e59628e58fe54929b9703bffd2641cfb 2024-11-14T02:45:51,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/8e6791f9902b4e518dd573e72cba9431 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/8e6791f9902b4e518dd573e72cba9431 2024-11-14T02:45:51,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/667fc89c86a845bc91c2671718bae26c to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/667fc89c86a845bc91c2671718bae26c 2024-11-14T02:45:51,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/d9300bee19b04daabfaa8b463fd2edc0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/d9300bee19b04daabfaa8b463fd2edc0 2024-11-14T02:45:51,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/6a0e0e65b016487f9a3934c011ccb014 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/6a0e0e65b016487f9a3934c011ccb014 2024-11-14T02:45:51,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/4c1a551b4aea4883b7beb1126acd26bd to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/4c1a551b4aea4883b7beb1126acd26bd 2024-11-14T02:45:51,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/60fe0493daba4cc5b89a9143256afdf5 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/60fe0493daba4cc5b89a9143256afdf5 2024-11-14T02:45:51,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/498a66f213bf4a138fc4e5e4c9f2ea11 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/498a66f213bf4a138fc4e5e4c9f2ea11 2024-11-14T02:45:51,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/629d69be753b4dcc9af632efa1c5bd4f to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/629d69be753b4dcc9af632efa1c5bd4f 2024-11-14T02:45:51,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/39b8da87fb564d2986417c477f9793a8 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/39b8da87fb564d2986417c477f9793a8 2024-11-14T02:45:51,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/b6f5bcf41f164b54b389856b9c5bd540 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/b6f5bcf41f164b54b389856b9c5bd540 2024-11-14T02:45:51,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/42a04109630e425792828abfc3d299da to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/42a04109630e425792828abfc3d299da 2024-11-14T02:45:51,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/7d101c15d9c44522af868edeffff9d65 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/7d101c15d9c44522af868edeffff9d65 2024-11-14T02:45:51,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/1767ec8de9a24aa2bc15c7fcaba8a55b to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/1767ec8de9a24aa2bc15c7fcaba8a55b 2024-11-14T02:45:51,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ff219687502744e28eb807f8baaf76fe, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/9ae05710bef3423babf92739ee5e7a30, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/e14249cbd6f44220a30e7128f81e9d7f, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/51df0d403aa844cdb4bab3dcba5bf1c6, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/8880cb3af9db4f47b72fc3f296c49dd3, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/c5790af34c264a10a943e34be0043503, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/b25fa1cef3d842d7b513a274b3b9fa43, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ddda86b7c2d34034abed79792d3bef58, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/3c5e6262858c4b82b777c12ba905600d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/050828cb6bb54f51831014804c75eefd, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/aa29140347704a5f930df48d5e4c8d8c, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/e336a7721f2b449da0656b60c0660bca, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/02d867b2fdaf4e308d8377ea0329a693, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/4ea1709fdd5f4fcfb17d215309d0603a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/8b3c413e0e00454ab8f5b257da287d8c, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/900af1c8ea384250bba524d35b6ecb8b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/2236343dec8d4840a54beeb157031be3, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/54d84d7baa404de18a157abe7b31d79b, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/345a6d6a69d843efb62604c4596a7aea, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ae04147a60164e069ef152f0531ae804, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/15d9411bb42a4fe296b5b5baa0af45ee, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/34cb2fb40edf4a47a3530cdb721de5b4, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/c70f1a22731944b1932fecde0600afb5, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ed89b8c645d147c48f0da8e67ba22eaf, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/1ae11521914843de918b6da83a5840f3, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/25af56cdf6864512a9a3a9ecb77293aa, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/929f51a56e3e4d2d9506de5830af8539, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ebefb91de92b4347bc367eab9d57cc0a] to archive 2024-11-14T02:45:51,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:45:51,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ff219687502744e28eb807f8baaf76fe to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ff219687502744e28eb807f8baaf76fe 2024-11-14T02:45:51,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/9ae05710bef3423babf92739ee5e7a30 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/9ae05710bef3423babf92739ee5e7a30 2024-11-14T02:45:51,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/e14249cbd6f44220a30e7128f81e9d7f to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/e14249cbd6f44220a30e7128f81e9d7f 2024-11-14T02:45:51,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/51df0d403aa844cdb4bab3dcba5bf1c6 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/51df0d403aa844cdb4bab3dcba5bf1c6 2024-11-14T02:45:51,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/8880cb3af9db4f47b72fc3f296c49dd3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/8880cb3af9db4f47b72fc3f296c49dd3 2024-11-14T02:45:51,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/c5790af34c264a10a943e34be0043503 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/c5790af34c264a10a943e34be0043503 2024-11-14T02:45:51,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/b25fa1cef3d842d7b513a274b3b9fa43 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/b25fa1cef3d842d7b513a274b3b9fa43 2024-11-14T02:45:51,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ddda86b7c2d34034abed79792d3bef58 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ddda86b7c2d34034abed79792d3bef58 2024-11-14T02:45:51,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/3c5e6262858c4b82b777c12ba905600d to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/3c5e6262858c4b82b777c12ba905600d 2024-11-14T02:45:51,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/050828cb6bb54f51831014804c75eefd to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/050828cb6bb54f51831014804c75eefd 2024-11-14T02:45:51,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/aa29140347704a5f930df48d5e4c8d8c to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/aa29140347704a5f930df48d5e4c8d8c 2024-11-14T02:45:51,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/e336a7721f2b449da0656b60c0660bca to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/e336a7721f2b449da0656b60c0660bca 2024-11-14T02:45:51,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/02d867b2fdaf4e308d8377ea0329a693 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/02d867b2fdaf4e308d8377ea0329a693 2024-11-14T02:45:51,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/4ea1709fdd5f4fcfb17d215309d0603a to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/4ea1709fdd5f4fcfb17d215309d0603a 2024-11-14T02:45:51,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/8b3c413e0e00454ab8f5b257da287d8c to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/8b3c413e0e00454ab8f5b257da287d8c 2024-11-14T02:45:51,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/900af1c8ea384250bba524d35b6ecb8b to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/900af1c8ea384250bba524d35b6ecb8b 2024-11-14T02:45:51,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/2236343dec8d4840a54beeb157031be3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/2236343dec8d4840a54beeb157031be3 2024-11-14T02:45:51,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/54d84d7baa404de18a157abe7b31d79b to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/54d84d7baa404de18a157abe7b31d79b 2024-11-14T02:45:51,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/345a6d6a69d843efb62604c4596a7aea to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/345a6d6a69d843efb62604c4596a7aea 2024-11-14T02:45:51,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ae04147a60164e069ef152f0531ae804 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ae04147a60164e069ef152f0531ae804 2024-11-14T02:45:51,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/15d9411bb42a4fe296b5b5baa0af45ee to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/15d9411bb42a4fe296b5b5baa0af45ee 2024-11-14T02:45:51,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/34cb2fb40edf4a47a3530cdb721de5b4 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/34cb2fb40edf4a47a3530cdb721de5b4 2024-11-14T02:45:51,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/c70f1a22731944b1932fecde0600afb5 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/c70f1a22731944b1932fecde0600afb5 2024-11-14T02:45:51,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ed89b8c645d147c48f0da8e67ba22eaf to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ed89b8c645d147c48f0da8e67ba22eaf 2024-11-14T02:45:51,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/1ae11521914843de918b6da83a5840f3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/1ae11521914843de918b6da83a5840f3 2024-11-14T02:45:51,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/25af56cdf6864512a9a3a9ecb77293aa to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/25af56cdf6864512a9a3a9ecb77293aa 2024-11-14T02:45:51,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/929f51a56e3e4d2d9506de5830af8539 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/929f51a56e3e4d2d9506de5830af8539 2024-11-14T02:45:51,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ebefb91de92b4347bc367eab9d57cc0a to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/ebefb91de92b4347bc367eab9d57cc0a 2024-11-14T02:45:51,220 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/recovered.edits/388.seqid, newMaxSeqId=388, maxSeqId=1 2024-11-14T02:45:51,220 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. 2024-11-14T02:45:51,220 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for 7922b12c74446660b37b836220447ee3: Waiting for close lock at 1731552349879Running coprocessor pre-close hooks at 1731552349879Disabling compacts and flushes for region at 1731552349879Disabling writes for close at 1731552349879Obtaining lock to block concurrent updates at 1731552349880 (+1 ms)Preparing flush snapshotting stores in 7922b12c74446660b37b836220447ee3 at 1731552349880Finished memstore snapshotting TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731552349880Flushing stores of TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3. at 1731552349882 (+2 ms)Flushing 7922b12c74446660b37b836220447ee3/A: creating writer at 1731552349882Flushing 7922b12c74446660b37b836220447ee3/A: appending metadata at 1731552349888 (+6 ms)Flushing 7922b12c74446660b37b836220447ee3/A: closing flushed file at 1731552349889 (+1 ms)Flushing 7922b12c74446660b37b836220447ee3/B: creating writer at 1731552350304 (+415 ms)Flushing 7922b12c74446660b37b836220447ee3/B: appending metadata at 1731552350309 (+5 ms)Flushing 7922b12c74446660b37b836220447ee3/B: closing flushed file at 1731552350309Flushing 7922b12c74446660b37b836220447ee3/C: creating writer at 1731552350724 (+415 ms)Flushing 7922b12c74446660b37b836220447ee3/C: appending metadata at 1731552350726 (+2 ms)Flushing 7922b12c74446660b37b836220447ee3/C: closing flushed file at 1731552350726Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59ef866c: reopening flushed file at 1731552351139 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4badb784: reopening flushed file at 1731552351144 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54145cbd: reopening flushed file at 1731552351148 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 7922b12c74446660b37b836220447ee3 in 1273ms, sequenceid=385, compaction requested=false at 1731552351152 (+4 ms)Writing region close event to WAL at 1731552351217 (+65 ms)Running coprocessor post-close hooks at 1731552351220 (+3 ms)Closed at 1731552351220 2024-11-14T02:45:51,222 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed 7922b12c74446660b37b836220447ee3 2024-11-14T02:45:51,223 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=7922b12c74446660b37b836220447ee3, regionState=CLOSED 2024-11-14T02:45:51,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 because future has completed 2024-11-14T02:45:51,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-14T02:45:51,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure 7922b12c74446660b37b836220447ee3, server=83c5a5c119d5,41455,1731552191259 in 1.5000 sec 2024-11-14T02:45:51,229 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-14T02:45:51,229 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7922b12c74446660b37b836220447ee3, UNASSIGN in 1.5050 sec 2024-11-14T02:45:51,231 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-14T02:45:51,231 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5090 sec 2024-11-14T02:45:51,232 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552351232"}]},"ts":"1731552351232"} 2024-11-14T02:45:51,233 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T02:45:51,233 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T02:45:51,235 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5210 sec 2024-11-14T02:45:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T02:45:51,865 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:51,867 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T02:45:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:51,872 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-14T02:45:51,874 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=139, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:51,878 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3 2024-11-14T02:45:51,882 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A, FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B, FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C, FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/recovered.edits] 2024-11-14T02:45:51,885 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/1a6e6eace70b42cdb0195c65ef13cd00 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/1a6e6eace70b42cdb0195c65ef13cd00 2024-11-14T02:45:51,886 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/f3dc6d3ace314a1ca1b9d9af07a170e2 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/A/f3dc6d3ace314a1ca1b9d9af07a170e2 2024-11-14T02:45:51,889 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/58738d8fae2f42d2a3dd7b58c80f6df6 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/58738d8fae2f42d2a3dd7b58c80f6df6 2024-11-14T02:45:51,890 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/f7934d3bab3c4a42a5a66412de6d8b18 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/B/f7934d3bab3c4a42a5a66412de6d8b18 2024-11-14T02:45:51,893 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/876197c18a25414fb4cc21da59c9ce03 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/876197c18a25414fb4cc21da59c9ce03 2024-11-14T02:45:51,895 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/bd427f78e71444aca6430801ba454b64 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/C/bd427f78e71444aca6430801ba454b64 2024-11-14T02:45:51,898 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/recovered.edits/388.seqid to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3/recovered.edits/388.seqid 2024-11-14T02:45:51,898 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/7922b12c74446660b37b836220447ee3 2024-11-14T02:45:51,898 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T02:45:51,900 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=139, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:51,902 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T02:45:51,904 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T02:45:51,904 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=139, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:51,904 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T02:45:51,905 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731552351904"}]},"ts":"9223372036854775807"} 2024-11-14T02:45:51,906 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T02:45:51,906 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7922b12c74446660b37b836220447ee3, NAME => 'TestAcidGuarantees,,1731552317417.7922b12c74446660b37b836220447ee3.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T02:45:51,906 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T02:45:51,906 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731552351906"}]},"ts":"9223372036854775807"} 2024-11-14T02:45:51,908 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T02:45:51,908 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=139, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:51,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 41 msec 2024-11-14T02:45:51,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-14T02:45:51,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T02:45:51,986 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:51,993 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=237 (was 238), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=281 (was 308), ProcessCount=11 (was 11), AvailableMemoryMB=5272 (was 5288) 2024-11-14T02:45:52,002 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=281, ProcessCount=11, AvailableMemoryMB=5272 2024-11-14T02:45:52,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T02:45:52,004 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T02:45:52,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:52,006 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T02:45:52,006 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:52,006 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 140 2024-11-14T02:45:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T02:45:52,007 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T02:45:52,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742310_1486 (size=959) 2024-11-14T02:45:52,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T02:45:52,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T02:45:52,419 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => eed6aa27f807a5ebe70b1551484242a3, NAME => 'TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b 2024-11-14T02:45:52,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742311_1487 (size=53) 2024-11-14T02:45:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T02:45:52,830 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:45:52,831 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing eed6aa27f807a5ebe70b1551484242a3, disabling compactions & flushes 2024-11-14T02:45:52,831 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:52,831 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:52,831 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. after waiting 0 ms 2024-11-14T02:45:52,831 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:52,831 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:52,831 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for eed6aa27f807a5ebe70b1551484242a3: Waiting for close lock at 1731552352831Disabling compacts and flushes for region at 1731552352831Disabling writes for close at 1731552352831Writing region close event to WAL at 1731552352831Closed at 1731552352831 2024-11-14T02:45:52,843 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T02:45:52,843 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731552352843"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731552352843"}]},"ts":"1731552352843"} 2024-11-14T02:45:52,846 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T02:45:52,848 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T02:45:52,848 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552352848"}]},"ts":"1731552352848"} 2024-11-14T02:45:52,851 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T02:45:52,852 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eed6aa27f807a5ebe70b1551484242a3, ASSIGN}] 2024-11-14T02:45:52,853 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eed6aa27f807a5ebe70b1551484242a3, ASSIGN 2024-11-14T02:45:52,854 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eed6aa27f807a5ebe70b1551484242a3, ASSIGN; state=OFFLINE, location=83c5a5c119d5,41455,1731552191259; forceNewPlan=false, retain=false 2024-11-14T02:45:53,005 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=eed6aa27f807a5ebe70b1551484242a3, regionState=OPENING, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:53,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eed6aa27f807a5ebe70b1551484242a3, ASSIGN because future has completed 2024-11-14T02:45:53,011 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259}] 2024-11-14T02:45:53,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T02:45:53,173 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:53,173 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7752): Opening region: {ENCODED => eed6aa27f807a5ebe70b1551484242a3, NAME => 'TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:45:53,174 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:53,174 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:45:53,174 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7794): checking encryption for eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:53,174 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7797): checking classloading for eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:53,176 INFO [StoreOpener-eed6aa27f807a5ebe70b1551484242a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:53,178 INFO [StoreOpener-eed6aa27f807a5ebe70b1551484242a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eed6aa27f807a5ebe70b1551484242a3 columnFamilyName A 2024-11-14T02:45:53,178 DEBUG [StoreOpener-eed6aa27f807a5ebe70b1551484242a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:53,178 INFO [StoreOpener-eed6aa27f807a5ebe70b1551484242a3-1 {}] regionserver.HStore(327): Store=eed6aa27f807a5ebe70b1551484242a3/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:45:53,179 INFO [StoreOpener-eed6aa27f807a5ebe70b1551484242a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:53,180 INFO [StoreOpener-eed6aa27f807a5ebe70b1551484242a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eed6aa27f807a5ebe70b1551484242a3 columnFamilyName B 2024-11-14T02:45:53,180 DEBUG [StoreOpener-eed6aa27f807a5ebe70b1551484242a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:53,181 INFO [StoreOpener-eed6aa27f807a5ebe70b1551484242a3-1 {}] regionserver.HStore(327): Store=eed6aa27f807a5ebe70b1551484242a3/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:45:53,181 INFO [StoreOpener-eed6aa27f807a5ebe70b1551484242a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:53,182 INFO [StoreOpener-eed6aa27f807a5ebe70b1551484242a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eed6aa27f807a5ebe70b1551484242a3 columnFamilyName C 2024-11-14T02:45:53,182 DEBUG [StoreOpener-eed6aa27f807a5ebe70b1551484242a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:53,183 INFO [StoreOpener-eed6aa27f807a5ebe70b1551484242a3-1 {}] regionserver.HStore(327): Store=eed6aa27f807a5ebe70b1551484242a3/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:45:53,183 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1038): replaying wal for eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:53,184 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:53,184 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:53,186 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1048): stopping wal replay for eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:53,186 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1060): Cleaning up temporary data for eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:53,186 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T02:45:53,188 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1093): writing seq id for eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:53,191 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T02:45:53,192 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1114): Opened eed6aa27f807a5ebe70b1551484242a3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68374078, jitterRate=0.01885315775871277}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T02:45:53,192 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:53,193 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1006): Region open journal for eed6aa27f807a5ebe70b1551484242a3: Running coprocessor pre-open hook at 1731552353175Writing region info on filesystem at 1731552353175Initializing all the Stores at 1731552353176 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552353176Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552353176Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552353176Cleaning up temporary data from old regions at 1731552353186 (+10 ms)Running coprocessor post-open hooks at 1731552353192 (+6 ms)Region opened successfully at 1731552353193 (+1 ms) 2024-11-14T02:45:53,194 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., pid=142, masterSystemTime=1731552353166 2024-11-14T02:45:53,197 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:53,197 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:53,197 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=eed6aa27f807a5ebe70b1551484242a3, regionState=OPEN, openSeqNum=2, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:53,199 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 because future has completed 2024-11-14T02:45:53,203 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-14T02:45:53,203 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 in 189 msec 2024-11-14T02:45:53,205 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-14T02:45:53,205 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eed6aa27f807a5ebe70b1551484242a3, ASSIGN in 351 msec 2024-11-14T02:45:53,205 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T02:45:53,206 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552353205"}]},"ts":"1731552353205"} 2024-11-14T02:45:53,207 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T02:45:53,208 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T02:45:53,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2040 sec 2024-11-14T02:45:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T02:45:54,154 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:54,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5581e3e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:54,156 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:45:54,156 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:54,158 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:45:54,158 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:45:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cbec9c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:45:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:54,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:54,161 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14c031f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:54,162 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:54,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:45:54,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:54,165 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:54,166 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T02:45:54,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 83c5a5c119d5,43239,1731552190584 2024-11-14T02:45:54,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6cf46752 2024-11-14T02:45:54,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T02:45:54,168 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T02:45:54,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T02:45:54,169 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T02:45:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:54,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742312_1488 (size=995) 2024-11-14T02:45:54,584 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-14T02:45:54,584 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-14T02:45:54,588 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T02:45:54,592 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eed6aa27f807a5ebe70b1551484242a3, REOPEN/MOVE}] 2024-11-14T02:45:54,593 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eed6aa27f807a5ebe70b1551484242a3, REOPEN/MOVE 2024-11-14T02:45:54,595 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=eed6aa27f807a5ebe70b1551484242a3, regionState=CLOSING, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:54,596 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eed6aa27f807a5ebe70b1551484242a3, REOPEN/MOVE because future has completed 2024-11-14T02:45:54,597 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T02:45:54,597 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259}] 2024-11-14T02:45:54,750 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:54,750 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T02:45:54,751 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing eed6aa27f807a5ebe70b1551484242a3, disabling compactions & flushes 2024-11-14T02:45:54,751 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:54,751 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:54,751 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. after waiting 0 ms 2024-11-14T02:45:54,751 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:54,759 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T02:45:54,760 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:54,760 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for eed6aa27f807a5ebe70b1551484242a3: Waiting for close lock at 1731552354750Running coprocessor pre-close hooks at 1731552354750Disabling compacts and flushes for region at 1731552354750Disabling writes for close at 1731552354751 (+1 ms)Writing region close event to WAL at 1731552354752 (+1 ms)Running coprocessor post-close hooks at 1731552354759 (+7 ms)Closed at 1731552354760 (+1 ms) 2024-11-14T02:45:54,760 WARN [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionServer(3299): Not adding moved region record: eed6aa27f807a5ebe70b1551484242a3 to self. 2024-11-14T02:45:54,762 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:54,763 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=eed6aa27f807a5ebe70b1551484242a3, regionState=CLOSED 2024-11-14T02:45:54,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 because future has completed 2024-11-14T02:45:54,769 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-14T02:45:54,769 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 in 169 msec 2024-11-14T02:45:54,770 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eed6aa27f807a5ebe70b1551484242a3, REOPEN/MOVE; state=CLOSED, location=83c5a5c119d5,41455,1731552191259; forceNewPlan=false, retain=true 2024-11-14T02:45:54,921 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=eed6aa27f807a5ebe70b1551484242a3, regionState=OPENING, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:54,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eed6aa27f807a5ebe70b1551484242a3, REOPEN/MOVE because future has completed 2024-11-14T02:45:54,924 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259}] 2024-11-14T02:45:55,084 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:55,085 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7752): Opening region: {ENCODED => eed6aa27f807a5ebe70b1551484242a3, NAME => 'TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:45:55,086 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:55,086 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:45:55,086 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7794): checking encryption for eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:55,086 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7797): checking classloading for eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:55,088 INFO [StoreOpener-eed6aa27f807a5ebe70b1551484242a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:55,089 INFO [StoreOpener-eed6aa27f807a5ebe70b1551484242a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eed6aa27f807a5ebe70b1551484242a3 columnFamilyName A 2024-11-14T02:45:55,090 DEBUG [StoreOpener-eed6aa27f807a5ebe70b1551484242a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:55,091 INFO [StoreOpener-eed6aa27f807a5ebe70b1551484242a3-1 {}] regionserver.HStore(327): Store=eed6aa27f807a5ebe70b1551484242a3/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:45:55,091 INFO [StoreOpener-eed6aa27f807a5ebe70b1551484242a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:55,092 INFO [StoreOpener-eed6aa27f807a5ebe70b1551484242a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eed6aa27f807a5ebe70b1551484242a3 columnFamilyName B 2024-11-14T02:45:55,092 DEBUG [StoreOpener-eed6aa27f807a5ebe70b1551484242a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:55,092 INFO [StoreOpener-eed6aa27f807a5ebe70b1551484242a3-1 {}] regionserver.HStore(327): Store=eed6aa27f807a5ebe70b1551484242a3/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:45:55,093 INFO [StoreOpener-eed6aa27f807a5ebe70b1551484242a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:55,093 INFO [StoreOpener-eed6aa27f807a5ebe70b1551484242a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eed6aa27f807a5ebe70b1551484242a3 columnFamilyName C 2024-11-14T02:45:55,093 DEBUG [StoreOpener-eed6aa27f807a5ebe70b1551484242a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:55,094 INFO [StoreOpener-eed6aa27f807a5ebe70b1551484242a3-1 {}] regionserver.HStore(327): Store=eed6aa27f807a5ebe70b1551484242a3/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:45:55,094 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1038): replaying wal for eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:55,095 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:55,096 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:55,097 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1048): stopping wal replay for eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:55,097 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1060): Cleaning up temporary data for eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:55,097 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T02:45:55,099 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1093): writing seq id for eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:55,100 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1114): Opened eed6aa27f807a5ebe70b1551484242a3; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72872218, jitterRate=0.08588066697120667}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T02:45:55,100 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:55,101 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1006): Region open journal for eed6aa27f807a5ebe70b1551484242a3: Running coprocessor pre-open hook at 1731552355086Writing region info on filesystem at 1731552355086Initializing all the Stores at 1731552355088 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552355088Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552355088Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552355088Cleaning up temporary data from old regions at 1731552355097 (+9 ms)Running coprocessor post-open hooks at 1731552355100 (+3 ms)Region opened successfully at 1731552355101 (+1 ms) 2024-11-14T02:45:55,102 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., pid=147, masterSystemTime=1731552355077 2024-11-14T02:45:55,104 DEBUG [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:55,104 INFO [RS_OPEN_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:55,105 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=eed6aa27f807a5ebe70b1551484242a3, regionState=OPEN, openSeqNum=5, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:55,106 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 because future has completed 2024-11-14T02:45:55,110 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=145 2024-11-14T02:45:55,110 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 in 183 msec 2024-11-14T02:45:55,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-14T02:45:55,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eed6aa27f807a5ebe70b1551484242a3, REOPEN/MOVE in 518 msec 2024-11-14T02:45:55,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-14T02:45:55,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 525 msec 2024-11-14T02:45:55,118 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 946 msec 2024-11-14T02:45:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-14T02:45:55,120 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:55,120 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1236062f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,120 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:45:55,121 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:55,121 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:45:55,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:55,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:45:55,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1931c61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:45:55,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:55,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:55,123 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:55,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f6e127d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,124 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a3e32a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,124 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:45:55,124 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:55,125 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:45:55,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:55,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:45:55,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ffeae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:45:55,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:55,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:55,126 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:55,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e57b00c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,127 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,127 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:45:55,127 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:55,128 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:45:55,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:55,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:45:55,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a945565, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:45:55,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:55,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:55,129 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:55,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d9064b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,130 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77590f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,130 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:45:55,130 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:55,130 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:45:55,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:55,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:45:55,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eb3efd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:45:55,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:55,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:55,131 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:55,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ba55204, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,132 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,132 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:45:55,132 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:55,133 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:45:55,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:55,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:45:55,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b90141, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:45:55,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:55,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:55,134 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:55,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d23f901, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,135 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6deae39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,135 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:45:55,135 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:55,136 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:45:55,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:55,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:45:55,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d7d825b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:45:55,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:55,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:55,137 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:55,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1644c693, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,138 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb7cff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,138 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:45:55,138 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:55,138 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:45:55,138 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:55,138 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:45:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4babc00b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:45:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:55,139 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:55,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69dee942, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,140 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,140 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:45:55,140 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:55,141 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:45:55,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:55,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:45:55,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1909e8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:45:55,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:55,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:55,142 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:55,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61967e39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,143 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39174ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,143 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:45:55,143 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:55,144 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:45:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:45:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11f088c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:45:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:55,145 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:55,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cf242aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,146 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8fc71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,146 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83c5a5c119d5,43239,-1 for getting cluster id 2024-11-14T02:45:55,146 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:55,146 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e010ce48-1cb1-4ad8-ae98-00527f341d4d' 2024-11-14T02:45:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e010ce48-1cb1-4ad8-ae98-00527f341d4d" 2024-11-14T02:45:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1181a4fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83c5a5c119d5,43239,-1] 2024-11-14T02:45:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:55,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:55,147 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:55,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@687a3c7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:55,148 DEBUG [Thread-2211 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:55,148 DEBUG [Thread-2209 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:55,148 DEBUG [Thread-2222 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:55,148 DEBUG [Thread-2213 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:55,148 DEBUG [Thread-2205 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:55,148 DEBUG [Thread-2218 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:55,149 DEBUG [Thread-2207 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:45:55,149 DEBUG [Thread-2209 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:45:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:45:55,149 DEBUG [Thread-2209 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:55,149 DEBUG [Thread-2224 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:45:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:45:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:45:55,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:55,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:55,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:55,150 DEBUG [Thread-2216 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:55,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:45:55,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:45:55,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:55,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:55,150 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:55,151 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:55,151 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:55,151 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:55,151 DEBUG [Thread-2220 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:55,151 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:55,151 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:55,151 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:55,151 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:45:55,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:55,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-14T02:45:55,152 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:55,152 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:55,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T02:45:55,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:45:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:45:55,153 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83c5a5c119d5,41455,1731552191259, seqNum=-1] 2024-11-14T02:45:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:45:55,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:45:55,155 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:45:55,155 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:45:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:45:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:45:55,156 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:45:55,156 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:55,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5] 2024-11-14T02:45:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:55,162 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:45:55,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eed6aa27f807a5ebe70b1551484242a3 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T02:45:55,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:55,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552415177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:55,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:55,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:55,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552415178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:55,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:55,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:55,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552415180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:55,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552415180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:55,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:55,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:55,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552415182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:55,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111486cda87a2b22473e83943c4378454e09_eed6aa27f807a5ebe70b1551484242a3 is 50, key is test_row_0/A:col10/1731552355160/Put/seqid=0 2024-11-14T02:45:55,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742313_1489 (size=12154) 2024-11-14T02:45:55,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T02:45:55,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:55,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552415293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:55,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:55,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552415293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:55,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:55,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:55,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552415293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:55,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552415293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:55,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552415293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:55,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T02:45:55,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:55,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. as already flushing 2024-11-14T02:45:55,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:55,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => eed6aa27f807a5ebe70b1551484242a3, NAME => 'TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:55,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => eed6aa27f807a5ebe70b1551484242a3, NAME => 'TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:55,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eed6aa27f807a5ebe70b1551484242a3, NAME => 'TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eed6aa27f807a5ebe70b1551484242a3, NAME => 'TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:55,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T02:45:55,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:55,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. as already flushing 2024-11-14T02:45:55,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:55,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => eed6aa27f807a5ebe70b1551484242a3, NAME => 'TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:55,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => eed6aa27f807a5ebe70b1551484242a3, NAME => 'TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eed6aa27f807a5ebe70b1551484242a3, NAME => 'TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eed6aa27f807a5ebe70b1551484242a3, NAME => 'TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T02:45:55,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552415503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:55,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:55,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552415503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:55,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:55,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552415503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:55,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:55,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552415503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:55,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:55,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552415503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:55,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:55,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:55,599 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:55,602 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111486cda87a2b22473e83943c4378454e09_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111486cda87a2b22473e83943c4378454e09_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:55,602 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/0b364ce6fcc64b6fab03df0b764e595d, store: [table=TestAcidGuarantees family=A region=eed6aa27f807a5ebe70b1551484242a3] 2024-11-14T02:45:55,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/0b364ce6fcc64b6fab03df0b764e595d is 175, key is test_row_0/A:col10/1731552355160/Put/seqid=0 2024-11-14T02:45:55,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742314_1490 (size=30955) 2024-11-14T02:45:55,607 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/0b364ce6fcc64b6fab03df0b764e595d 2024-11-14T02:45:55,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T02:45:55,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:55,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. as already flushing 2024-11-14T02:45:55,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:55,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => eed6aa27f807a5ebe70b1551484242a3, NAME => 'TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:55,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => eed6aa27f807a5ebe70b1551484242a3, NAME => 'TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eed6aa27f807a5ebe70b1551484242a3, NAME => 'TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eed6aa27f807a5ebe70b1551484242a3, NAME => 'TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:55,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/a8d4db196152456f809396385630d025 is 50, key is test_row_0/B:col10/1731552355160/Put/seqid=0 2024-11-14T02:45:55,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742315_1491 (size=12001) 2024-11-14T02:45:55,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T02:45:55,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:55,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. as already flushing 2024-11-14T02:45:55,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:55,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => eed6aa27f807a5ebe70b1551484242a3, NAME => 'TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:55,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => eed6aa27f807a5ebe70b1551484242a3, NAME => 'TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eed6aa27f807a5ebe70b1551484242a3, NAME => 'TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eed6aa27f807a5ebe70b1551484242a3, NAME => 'TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T02:45:55,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:55,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552415813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552415813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:55,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552415813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552415813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552415813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:55,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T02:45:55,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:55,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. as already flushing 2024-11-14T02:45:55,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:55,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => eed6aa27f807a5ebe70b1551484242a3, NAME => 'TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:55,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => eed6aa27f807a5ebe70b1551484242a3, NAME => 'TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eed6aa27f807a5ebe70b1551484242a3, NAME => 'TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eed6aa27f807a5ebe70b1551484242a3, NAME => 'TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:56,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/a8d4db196152456f809396385630d025 2024-11-14T02:45:56,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/cee87745477244acb23f6d2fd4db9690 is 50, key is test_row_0/C:col10/1731552355160/Put/seqid=0 2024-11-14T02:45:56,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742316_1492 (size=12001) 2024-11-14T02:45:56,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/cee87745477244acb23f6d2fd4db9690 2024-11-14T02:45:56,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/0b364ce6fcc64b6fab03df0b764e595d as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/0b364ce6fcc64b6fab03df0b764e595d 2024-11-14T02:45:56,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/0b364ce6fcc64b6fab03df0b764e595d, entries=150, sequenceid=17, filesize=30.2 K 2024-11-14T02:45:56,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/a8d4db196152456f809396385630d025 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/a8d4db196152456f809396385630d025 2024-11-14T02:45:56,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/a8d4db196152456f809396385630d025, entries=150, sequenceid=17, filesize=11.7 K 2024-11-14T02:45:56,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/cee87745477244acb23f6d2fd4db9690 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/cee87745477244acb23f6d2fd4db9690 2024-11-14T02:45:56,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/cee87745477244acb23f6d2fd4db9690, entries=150, sequenceid=17, filesize=11.7 K 2024-11-14T02:45:56,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for eed6aa27f807a5ebe70b1551484242a3 in 902ms, sequenceid=17, compaction requested=false 2024-11-14T02:45:56,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eed6aa27f807a5ebe70b1551484242a3: 2024-11-14T02:45:56,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T02:45:56,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:56,074 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing eed6aa27f807a5ebe70b1551484242a3 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-14T02:45:56,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111469c4bafc403a4549a806b2d1d13df8ec_eed6aa27f807a5ebe70b1551484242a3 is 50, key is test_row_0/A:col10/1731552355176/Put/seqid=0 2024-11-14T02:45:56,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742317_1493 (size=12154) 2024-11-14T02:45:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T02:45:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:56,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. as already flushing 2024-11-14T02:45:56,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552416332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:56,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:56,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552416333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552416333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552416334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552416334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:56,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:56,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552416443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:56,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:56,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552416443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:56,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:56,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552416443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:56,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552416443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:56,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552416443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:56,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:56,486 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111469c4bafc403a4549a806b2d1d13df8ec_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111469c4bafc403a4549a806b2d1d13df8ec_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:56,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/b3348773285946698a8003a7d908a9a4, store: [table=TestAcidGuarantees family=A region=eed6aa27f807a5ebe70b1551484242a3] 2024-11-14T02:45:56,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/b3348773285946698a8003a7d908a9a4 is 175, key is test_row_0/A:col10/1731552355176/Put/seqid=0 2024-11-14T02:45:56,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742318_1494 (size=30955) 2024-11-14T02:45:56,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552416653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:56,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552416653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552416653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552416653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:56,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552416653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:56,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:56,743 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T02:45:56,891 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/b3348773285946698a8003a7d908a9a4 2024-11-14T02:45:56,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/6b8ede0abbd040c1b7fccc427e73adb9 is 50, key is test_row_0/B:col10/1731552355176/Put/seqid=0 2024-11-14T02:45:56,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742319_1495 (size=12001) 2024-11-14T02:45:56,909 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/6b8ede0abbd040c1b7fccc427e73adb9 2024-11-14T02:45:56,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/3100a2f1f0724d6db149cec8199b6a54 is 50, key is test_row_0/C:col10/1731552355176/Put/seqid=0 2024-11-14T02:45:56,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742320_1496 (size=12001) 2024-11-14T02:45:56,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552416963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552416963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:56,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552416963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552416963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:56,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552416963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T02:45:57,321 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/3100a2f1f0724d6db149cec8199b6a54 2024-11-14T02:45:57,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/b3348773285946698a8003a7d908a9a4 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/b3348773285946698a8003a7d908a9a4 2024-11-14T02:45:57,327 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/b3348773285946698a8003a7d908a9a4, entries=150, sequenceid=40, filesize=30.2 K 2024-11-14T02:45:57,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/6b8ede0abbd040c1b7fccc427e73adb9 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/6b8ede0abbd040c1b7fccc427e73adb9 2024-11-14T02:45:57,337 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/6b8ede0abbd040c1b7fccc427e73adb9, entries=150, sequenceid=40, filesize=11.7 K 2024-11-14T02:45:57,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/3100a2f1f0724d6db149cec8199b6a54 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/3100a2f1f0724d6db149cec8199b6a54 2024-11-14T02:45:57,341 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/3100a2f1f0724d6db149cec8199b6a54, entries=150, sequenceid=40, filesize=11.7 K 2024-11-14T02:45:57,342 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for eed6aa27f807a5ebe70b1551484242a3 in 1268ms, sequenceid=40, compaction requested=false 2024-11-14T02:45:57,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for eed6aa27f807a5ebe70b1551484242a3: 2024-11-14T02:45:57,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:57,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-14T02:45:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-14T02:45:57,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-14T02:45:57,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1870 sec 2024-11-14T02:45:57,346 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.1950 sec 2024-11-14T02:45:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:57,475 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:45:57,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eed6aa27f807a5ebe70b1551484242a3 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-14T02:45:57,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a3045b68bdd84b87b028e2482c6c55b9_eed6aa27f807a5ebe70b1551484242a3 is 50, key is test_row_0/A:col10/1731552357475/Put/seqid=0 2024-11-14T02:45:57,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742321_1497 (size=12154) 2024-11-14T02:45:57,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552417492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:57,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:57,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552417493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:57,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:57,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552417495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:57,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:57,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552417496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552417496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:57,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552417603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:57,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:57,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552417603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:57,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:57,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552417603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:57,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:57,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552417603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:57,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:57,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552417604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:57,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:57,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552417813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:57,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:57,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552417813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:57,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552417813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:57,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552417814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:57,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552417814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:57,886 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:57,918 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a3045b68bdd84b87b028e2482c6c55b9_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a3045b68bdd84b87b028e2482c6c55b9_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:57,918 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/9c3d3d12ff364543b6d52a1920df0dbb, store: [table=TestAcidGuarantees family=A region=eed6aa27f807a5ebe70b1551484242a3] 2024-11-14T02:45:57,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/9c3d3d12ff364543b6d52a1920df0dbb is 175, key is test_row_0/A:col10/1731552357475/Put/seqid=0 2024-11-14T02:45:57,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742322_1498 (size=30955) 2024-11-14T02:45:58,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:58,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552418123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:58,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552418123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:58,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552418123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:58,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:58,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552418123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:58,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:58,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:58,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552418123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:58,323 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/9c3d3d12ff364543b6d52a1920df0dbb 2024-11-14T02:45:58,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/fc9ff8dbcafb4dd782e549d0fa4ff9ed is 50, key is test_row_0/B:col10/1731552357475/Put/seqid=0 2024-11-14T02:45:58,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742323_1499 (size=12001) 2024-11-14T02:45:58,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:58,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552418633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:58,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552418633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:58,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552418633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:58,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552418633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:58,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552418633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:58,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:58,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/fc9ff8dbcafb4dd782e549d0fa4ff9ed 2024-11-14T02:45:58,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/c6dba4a94b28458690be152211c6958a is 50, key is test_row_0/C:col10/1731552357475/Put/seqid=0 2024-11-14T02:45:58,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742324_1500 (size=12001) 2024-11-14T02:45:58,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/c6dba4a94b28458690be152211c6958a 2024-11-14T02:45:58,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/9c3d3d12ff364543b6d52a1920df0dbb as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/9c3d3d12ff364543b6d52a1920df0dbb 2024-11-14T02:45:58,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/9c3d3d12ff364543b6d52a1920df0dbb, entries=150, sequenceid=56, filesize=30.2 K 2024-11-14T02:45:58,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/fc9ff8dbcafb4dd782e549d0fa4ff9ed as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/fc9ff8dbcafb4dd782e549d0fa4ff9ed 2024-11-14T02:45:58,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/fc9ff8dbcafb4dd782e549d0fa4ff9ed, entries=150, sequenceid=56, filesize=11.7 K 2024-11-14T02:45:58,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/c6dba4a94b28458690be152211c6958a as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/c6dba4a94b28458690be152211c6958a 2024-11-14T02:45:58,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/c6dba4a94b28458690be152211c6958a, entries=150, sequenceid=56, filesize=11.7 K 2024-11-14T02:45:58,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for eed6aa27f807a5ebe70b1551484242a3 in 1293ms, sequenceid=56, compaction requested=true 2024-11-14T02:45:58,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eed6aa27f807a5ebe70b1551484242a3: 2024-11-14T02:45:58,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eed6aa27f807a5ebe70b1551484242a3:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:45:58,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:58,768 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:58,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eed6aa27f807a5ebe70b1551484242a3:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:45:58,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:58,768 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:58,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eed6aa27f807a5ebe70b1551484242a3:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:45:58,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:58,769 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:58,769 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): eed6aa27f807a5ebe70b1551484242a3/A is initiating minor compaction (all files) 2024-11-14T02:45:58,769 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eed6aa27f807a5ebe70b1551484242a3/A in TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:58,769 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/0b364ce6fcc64b6fab03df0b764e595d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/b3348773285946698a8003a7d908a9a4, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/9c3d3d12ff364543b6d52a1920df0dbb] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp, totalSize=90.7 K 2024-11-14T02:45:58,769 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:58,769 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. files: [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/0b364ce6fcc64b6fab03df0b764e595d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/b3348773285946698a8003a7d908a9a4, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/9c3d3d12ff364543b6d52a1920df0dbb] 2024-11-14T02:45:58,769 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:58,769 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): eed6aa27f807a5ebe70b1551484242a3/B is initiating minor compaction (all files) 2024-11-14T02:45:58,769 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eed6aa27f807a5ebe70b1551484242a3/B in TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:58,770 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/a8d4db196152456f809396385630d025, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/6b8ede0abbd040c1b7fccc427e73adb9, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/fc9ff8dbcafb4dd782e549d0fa4ff9ed] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp, totalSize=35.2 K 2024-11-14T02:45:58,770 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b364ce6fcc64b6fab03df0b764e595d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731552355160 2024-11-14T02:45:58,770 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting a8d4db196152456f809396385630d025, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731552355160 2024-11-14T02:45:58,770 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b8ede0abbd040c1b7fccc427e73adb9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731552355176 2024-11-14T02:45:58,770 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3348773285946698a8003a7d908a9a4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731552355176 2024-11-14T02:45:58,771 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting fc9ff8dbcafb4dd782e549d0fa4ff9ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731552356332 2024-11-14T02:45:58,771 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c3d3d12ff364543b6d52a1920df0dbb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731552356332 2024-11-14T02:45:58,776 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=eed6aa27f807a5ebe70b1551484242a3] 2024-11-14T02:45:58,777 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eed6aa27f807a5ebe70b1551484242a3#B#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:58,777 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/a9fd589f8a744d67a2eac84d25278810 is 50, key is test_row_0/B:col10/1731552357475/Put/seqid=0 2024-11-14T02:45:58,779 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111445ebb7fac70a4da3b9420926cb4cb974_eed6aa27f807a5ebe70b1551484242a3 store=[table=TestAcidGuarantees family=A region=eed6aa27f807a5ebe70b1551484242a3] 2024-11-14T02:45:58,781 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111445ebb7fac70a4da3b9420926cb4cb974_eed6aa27f807a5ebe70b1551484242a3, store=[table=TestAcidGuarantees family=A region=eed6aa27f807a5ebe70b1551484242a3] 2024-11-14T02:45:58,781 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111445ebb7fac70a4da3b9420926cb4cb974_eed6aa27f807a5ebe70b1551484242a3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eed6aa27f807a5ebe70b1551484242a3] 2024-11-14T02:45:58,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742325_1501 (size=12104) 2024-11-14T02:45:58,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742326_1502 (size=4469) 2024-11-14T02:45:59,187 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/a9fd589f8a744d67a2eac84d25278810 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/a9fd589f8a744d67a2eac84d25278810 2024-11-14T02:45:59,189 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eed6aa27f807a5ebe70b1551484242a3#A#compaction#425 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:59,190 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/b209a940ce75415ca6ad0714bdbb28ea is 175, key is test_row_0/A:col10/1731552357475/Put/seqid=0 2024-11-14T02:45:59,191 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eed6aa27f807a5ebe70b1551484242a3/B of eed6aa27f807a5ebe70b1551484242a3 into a9fd589f8a744d67a2eac84d25278810(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:59,191 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eed6aa27f807a5ebe70b1551484242a3: 2024-11-14T02:45:59,191 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., storeName=eed6aa27f807a5ebe70b1551484242a3/B, priority=13, startTime=1731552358768; duration=0sec 2024-11-14T02:45:59,191 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:59,191 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eed6aa27f807a5ebe70b1551484242a3:B 2024-11-14T02:45:59,191 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:59,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742327_1503 (size=31058) 2024-11-14T02:45:59,196 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:59,196 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): eed6aa27f807a5ebe70b1551484242a3/C is initiating minor compaction (all files) 2024-11-14T02:45:59,196 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eed6aa27f807a5ebe70b1551484242a3/C in TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:59,196 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/cee87745477244acb23f6d2fd4db9690, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/3100a2f1f0724d6db149cec8199b6a54, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/c6dba4a94b28458690be152211c6958a] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp, totalSize=35.2 K 2024-11-14T02:45:59,196 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting cee87745477244acb23f6d2fd4db9690, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731552355160 2024-11-14T02:45:59,196 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 3100a2f1f0724d6db149cec8199b6a54, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731552355176 2024-11-14T02:45:59,197 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting c6dba4a94b28458690be152211c6958a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731552356332 2024-11-14T02:45:59,202 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eed6aa27f807a5ebe70b1551484242a3#C#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:59,203 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/f367540fdfcd4b0da2d7d98a71e0ad97 is 50, key is test_row_0/C:col10/1731552357475/Put/seqid=0 2024-11-14T02:45:59,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742328_1504 (size=12104) 2024-11-14T02:45:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T02:45:59,314 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:59,314 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:59,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-14T02:45:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T02:45:59,316 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:59,317 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:59,317 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:59,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T02:45:59,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T02:45:59,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:45:59,469 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing eed6aa27f807a5ebe70b1551484242a3 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-14T02:45:59,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144d4c2744f77f4489bc2217d17db3edab_eed6aa27f807a5ebe70b1551484242a3 is 50, key is test_row_0/A:col10/1731552357492/Put/seqid=0 2024-11-14T02:45:59,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742329_1505 (size=12154) 2024-11-14T02:45:59,599 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/b209a940ce75415ca6ad0714bdbb28ea as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/b209a940ce75415ca6ad0714bdbb28ea 2024-11-14T02:45:59,602 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eed6aa27f807a5ebe70b1551484242a3/A of eed6aa27f807a5ebe70b1551484242a3 into b209a940ce75415ca6ad0714bdbb28ea(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:59,602 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eed6aa27f807a5ebe70b1551484242a3: 2024-11-14T02:45:59,602 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., storeName=eed6aa27f807a5ebe70b1551484242a3/A, priority=13, startTime=1731552358768; duration=0sec 2024-11-14T02:45:59,603 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:59,603 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eed6aa27f807a5ebe70b1551484242a3:A 2024-11-14T02:45:59,617 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/f367540fdfcd4b0da2d7d98a71e0ad97 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/f367540fdfcd4b0da2d7d98a71e0ad97 2024-11-14T02:45:59,622 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eed6aa27f807a5ebe70b1551484242a3/C of eed6aa27f807a5ebe70b1551484242a3 into f367540fdfcd4b0da2d7d98a71e0ad97(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:59,622 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eed6aa27f807a5ebe70b1551484242a3: 2024-11-14T02:45:59,622 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., storeName=eed6aa27f807a5ebe70b1551484242a3/C, priority=13, startTime=1731552358768; duration=0sec 2024-11-14T02:45:59,622 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:59,623 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eed6aa27f807a5ebe70b1551484242a3:C 2024-11-14T02:45:59,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T02:45:59,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:59,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. as already flushing 2024-11-14T02:45:59,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:59,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552419658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:59,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:59,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:59,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552419659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:59,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:59,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:59,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552419661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:59,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:59,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:59,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552419662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:59,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:59,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:59,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552419662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:59,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:59,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:59,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:59,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552419773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:59,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552419773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:59,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:59,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552419773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:59,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:59,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:59,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552419774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:59,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:59,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552419774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:59,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:59,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:59,881 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144d4c2744f77f4489bc2217d17db3edab_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144d4c2744f77f4489bc2217d17db3edab_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:45:59,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/a165b5285f5a45528bca3cc7e1e242a0, store: [table=TestAcidGuarantees family=A region=eed6aa27f807a5ebe70b1551484242a3] 2024-11-14T02:45:59,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/a165b5285f5a45528bca3cc7e1e242a0 is 175, key is test_row_0/A:col10/1731552357492/Put/seqid=0 2024-11-14T02:45:59,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742330_1506 (size=30955) 2024-11-14T02:45:59,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T02:45:59,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552419983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:59,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552419983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:59,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552419984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:59,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552419984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:45:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552419984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:45:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:00,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:00,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552420293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:00,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:00,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:00,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:00,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:00,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:00,297 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/a165b5285f5a45528bca3cc7e1e242a0 2024-11-14T02:46:00,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552420293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:00,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552420293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:00,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:00,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:00,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:00,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:00,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:00,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:00,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:00,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:00,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552420293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:00,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552420293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:00,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:00,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:00,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:00,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:00,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:00,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:00,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/2d1bf29696cf40f8acfb601f44e829b0 is 50, key is test_row_0/B:col10/1731552357492/Put/seqid=0 2024-11-14T02:46:00,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742331_1507 (size=12001) 2024-11-14T02:46:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T02:46:00,708 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/2d1bf29696cf40f8acfb601f44e829b0 2024-11-14T02:46:00,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/bdc5f0f05ca149ba84014e7ec40d03e8 is 50, key is test_row_0/C:col10/1731552357492/Put/seqid=0 2024-11-14T02:46:00,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742332_1508 (size=12001) 2024-11-14T02:46:00,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552420803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:00,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552420803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:00,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552420803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:00,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552420803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:00,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:00,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552420803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:01,118 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/bdc5f0f05ca149ba84014e7ec40d03e8 2024-11-14T02:46:01,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/a165b5285f5a45528bca3cc7e1e242a0 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/a165b5285f5a45528bca3cc7e1e242a0 2024-11-14T02:46:01,124 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/a165b5285f5a45528bca3cc7e1e242a0, entries=150, sequenceid=78, filesize=30.2 K 2024-11-14T02:46:01,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/2d1bf29696cf40f8acfb601f44e829b0 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/2d1bf29696cf40f8acfb601f44e829b0 2024-11-14T02:46:01,127 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/2d1bf29696cf40f8acfb601f44e829b0, entries=150, sequenceid=78, filesize=11.7 K 2024-11-14T02:46:01,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/bdc5f0f05ca149ba84014e7ec40d03e8 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/bdc5f0f05ca149ba84014e7ec40d03e8 2024-11-14T02:46:01,130 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/bdc5f0f05ca149ba84014e7ec40d03e8, entries=150, sequenceid=78, filesize=11.7 K 2024-11-14T02:46:01,131 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for eed6aa27f807a5ebe70b1551484242a3 in 1662ms, sequenceid=78, compaction requested=false 2024-11-14T02:46:01,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for eed6aa27f807a5ebe70b1551484242a3: 2024-11-14T02:46:01,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:46:01,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-14T02:46:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-14T02:46:01,133 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-14T02:46:01,133 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8150 sec 2024-11-14T02:46:01,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.8200 sec 2024-11-14T02:46:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T02:46:01,463 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:46:01,464 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:46:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-14T02:46:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T02:46:01,465 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:46:01,466 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:46:01,466 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:46:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T02:46:01,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T02:46:01,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:46:01,618 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing eed6aa27f807a5ebe70b1551484242a3 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-14T02:46:01,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145ef3a1e938cf41e0b2dc96746aaa12e5_eed6aa27f807a5ebe70b1551484242a3 is 50, key is test_row_0/A:col10/1731552359655/Put/seqid=0 2024-11-14T02:46:01,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742333_1509 (size=12154) 2024-11-14T02:46:01,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T02:46:01,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:01,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. as already flushing 2024-11-14T02:46:01,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:01,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552421829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:01,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:01,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:01,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:01,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552421830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:01,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:01,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:01,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:01,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:01,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:01,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552421831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:01,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:01,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:01,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:01,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:01,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552421832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:01,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:01,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:01,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:01,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:01,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552421833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:01,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:01,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:01,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:01,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552421943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:01,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552421943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:01,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552421943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:01,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:01,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552421944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552421944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:02,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:02,030 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145ef3a1e938cf41e0b2dc96746aaa12e5_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145ef3a1e938cf41e0b2dc96746aaa12e5_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:02,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/5cfee9372ef6490687b37948288912f3, store: [table=TestAcidGuarantees family=A region=eed6aa27f807a5ebe70b1551484242a3] 2024-11-14T02:46:02,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/5cfee9372ef6490687b37948288912f3 is 175, key is test_row_0/A:col10/1731552359655/Put/seqid=0 2024-11-14T02:46:02,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742334_1510 (size=30955) 2024-11-14T02:46:02,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T02:46:02,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:02,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552422154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:02,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:02,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552422154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:02,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:02,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:02,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:02,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552422154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:02,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:02,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:02,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552422154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:02,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:02,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:02,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552422154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:02,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:02,435 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/5cfee9372ef6490687b37948288912f3 2024-11-14T02:46:02,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/6234968d05e848debb344d17f8313299 is 50, key is test_row_0/B:col10/1731552359655/Put/seqid=0 2024-11-14T02:46:02,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742335_1511 (size=12001) 2024-11-14T02:46:02,448 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/6234968d05e848debb344d17f8313299 2024-11-14T02:46:02,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/05abe55e8cbf45e9955881378940f3e5 is 50, key is test_row_0/C:col10/1731552359655/Put/seqid=0 2024-11-14T02:46:02,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742336_1512 (size=12001) 2024-11-14T02:46:02,462 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/05abe55e8cbf45e9955881378940f3e5 2024-11-14T02:46:02,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:02,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552422463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:02,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:02,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552422464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:02,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:02,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:02,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/5cfee9372ef6490687b37948288912f3 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/5cfee9372ef6490687b37948288912f3 2024-11-14T02:46:02,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:02,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552422464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:02,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:02,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:02,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552422464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:02,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:02,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:02,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552422464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:02,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:02,471 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/5cfee9372ef6490687b37948288912f3, entries=150, sequenceid=96, filesize=30.2 K 2024-11-14T02:46:02,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/6234968d05e848debb344d17f8313299 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/6234968d05e848debb344d17f8313299 2024-11-14T02:46:02,475 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/6234968d05e848debb344d17f8313299, entries=150, sequenceid=96, filesize=11.7 K 2024-11-14T02:46:02,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/05abe55e8cbf45e9955881378940f3e5 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/05abe55e8cbf45e9955881378940f3e5 2024-11-14T02:46:02,479 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/05abe55e8cbf45e9955881378940f3e5, entries=150, sequenceid=96, filesize=11.7 K 2024-11-14T02:46:02,480 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for eed6aa27f807a5ebe70b1551484242a3 in 862ms, sequenceid=96, compaction requested=true 2024-11-14T02:46:02,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for eed6aa27f807a5ebe70b1551484242a3: 2024-11-14T02:46:02,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:46:02,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-14T02:46:02,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-14T02:46:02,483 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-14T02:46:02,483 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0160 sec 2024-11-14T02:46:02,485 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.0200 sec 2024-11-14T02:46:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T02:46:02,603 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:46:02,603 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:46:02,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-14T02:46:02,605 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:46:02,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T02:46:02,606 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:46:02,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:46:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T02:46:02,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-14T02:46:02,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:46:02,758 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing eed6aa27f807a5ebe70b1551484242a3 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-14T02:46:02,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114429460c4648e483db3b213fcca9598ba_eed6aa27f807a5ebe70b1551484242a3 is 50, key is test_row_0/A:col10/1731552361829/Put/seqid=0 2024-11-14T02:46:02,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742337_1513 (size=12154) 2024-11-14T02:46:02,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T02:46:02,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:02,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. as already flushing 2024-11-14T02:46:02,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:02,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552422984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:02,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:02,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552422986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:02,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552422986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:02,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:02,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:02,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552422989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:02,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552422989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:03,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552423093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:03,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:03,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552423093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:03,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:03,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552423093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:03,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:03,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:03,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:03,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552423103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:03,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:03,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:03,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552423104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:03,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:03,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:03,177 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114429460c4648e483db3b213fcca9598ba_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114429460c4648e483db3b213fcca9598ba_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:03,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/4a55b7b265934843afc883a28e8fdbee, store: [table=TestAcidGuarantees family=A region=eed6aa27f807a5ebe70b1551484242a3] 2024-11-14T02:46:03,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/4a55b7b265934843afc883a28e8fdbee is 175, key is test_row_0/A:col10/1731552361829/Put/seqid=0 2024-11-14T02:46:03,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742338_1514 (size=30955) 2024-11-14T02:46:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T02:46:03,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552423303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:03,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:03,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552423303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:03,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:03,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552423303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:03,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:03,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:03,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552423314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:03,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:03,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:03,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552423314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:03,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:03,587 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/4a55b7b265934843afc883a28e8fdbee 2024-11-14T02:46:03,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/c9f24f0658be47c08ebe98b3dfb405e2 is 50, key is test_row_0/B:col10/1731552361829/Put/seqid=0 2024-11-14T02:46:03,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742339_1515 (size=12001) 2024-11-14T02:46:03,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:03,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552423613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:03,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:03,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552423613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:03,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:03,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552423613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:03,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552423623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:03,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:03,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552423623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:03,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:03,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:03,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T02:46:03,997 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/c9f24f0658be47c08ebe98b3dfb405e2 2024-11-14T02:46:04,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/f6924a074fa04038b402970113b15592 is 50, key is test_row_0/C:col10/1731552361829/Put/seqid=0 2024-11-14T02:46:04,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742340_1516 (size=12001) 2024-11-14T02:46:04,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:04,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552424123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:04,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:04,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552424123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:04,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:04,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552424123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:04,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:04,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552424133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:04,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:04,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552424133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:04,406 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/f6924a074fa04038b402970113b15592 2024-11-14T02:46:04,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/4a55b7b265934843afc883a28e8fdbee as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/4a55b7b265934843afc883a28e8fdbee 2024-11-14T02:46:04,411 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/4a55b7b265934843afc883a28e8fdbee, entries=150, sequenceid=116, filesize=30.2 K 2024-11-14T02:46:04,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/c9f24f0658be47c08ebe98b3dfb405e2 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/c9f24f0658be47c08ebe98b3dfb405e2 2024-11-14T02:46:04,414 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/c9f24f0658be47c08ebe98b3dfb405e2, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T02:46:04,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/f6924a074fa04038b402970113b15592 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/f6924a074fa04038b402970113b15592 2024-11-14T02:46:04,418 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/f6924a074fa04038b402970113b15592, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T02:46:04,418 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for eed6aa27f807a5ebe70b1551484242a3 in 1660ms, sequenceid=116, compaction requested=true 2024-11-14T02:46:04,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for eed6aa27f807a5ebe70b1551484242a3: 2024-11-14T02:46:04,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:46:04,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-14T02:46:04,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-14T02:46:04,421 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-14T02:46:04,421 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8140 sec 2024-11-14T02:46:04,422 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.8180 sec 2024-11-14T02:46:04,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T02:46:04,753 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:46:04,754 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:46:04,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-14T02:46:04,755 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:46:04,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T02:46:04,755 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:46:04,756 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:46:04,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T02:46:04,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T02:46:04,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:46:04,908 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing eed6aa27f807a5ebe70b1551484242a3 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-14T02:46:04,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143d182453eb2447beaf8ecad4f390f73b_eed6aa27f807a5ebe70b1551484242a3 is 50, key is test_row_0/A:col10/1731552362988/Put/seqid=0 2024-11-14T02:46:04,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742341_1517 (size=12254) 2024-11-14T02:46:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T02:46:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:05,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. as already flushing 2024-11-14T02:46:05,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552425153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:05,158 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_0' in eed6aa27f807a5ebe70b1551484242a3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:46:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552425154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:05,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552425156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552425157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552425158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:05,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552425263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:05,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552425274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:05,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552425274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:05,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:05,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:05,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552425274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:05,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:05,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:05,323 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143d182453eb2447beaf8ecad4f390f73b_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143d182453eb2447beaf8ecad4f390f73b_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:05,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/ffecef329e374a29b5f7270d4e7572b8, store: [table=TestAcidGuarantees family=A region=eed6aa27f807a5ebe70b1551484242a3] 2024-11-14T02:46:05,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/ffecef329e374a29b5f7270d4e7572b8 is 175, key is test_row_0/A:col10/1731552362988/Put/seqid=0 2024-11-14T02:46:05,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742342_1518 (size=31055) 2024-11-14T02:46:05,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T02:46:05,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:05,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552425473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:05,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:05,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552425483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:05,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552425483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:05,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552425483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:05,728 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/ffecef329e374a29b5f7270d4e7572b8 2024-11-14T02:46:05,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/15ae4822a1dd490daa91fca227d5459a is 50, key is test_row_0/B:col10/1731552362988/Put/seqid=0 2024-11-14T02:46:05,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742343_1519 (size=12101) 2024-11-14T02:46:05,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:05,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552425783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:05,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:05,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552425793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:05,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552425793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552425793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T02:46:06,137 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/15ae4822a1dd490daa91fca227d5459a 2024-11-14T02:46:06,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/c02800042c1444309d95a7c8dd26a69d is 50, key is test_row_0/C:col10/1731552362988/Put/seqid=0 2024-11-14T02:46:06,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742344_1520 (size=12101) 2024-11-14T02:46:06,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552426293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:06,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552426303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:06,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:06,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:06,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:06,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552426303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:06,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:06,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:06,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:06,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552426304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:06,546 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/c02800042c1444309d95a7c8dd26a69d 2024-11-14T02:46:06,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/ffecef329e374a29b5f7270d4e7572b8 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/ffecef329e374a29b5f7270d4e7572b8 2024-11-14T02:46:06,552 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/ffecef329e374a29b5f7270d4e7572b8, entries=150, sequenceid=133, filesize=30.3 K 2024-11-14T02:46:06,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/15ae4822a1dd490daa91fca227d5459a as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/15ae4822a1dd490daa91fca227d5459a 2024-11-14T02:46:06,555 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/15ae4822a1dd490daa91fca227d5459a, entries=150, sequenceid=133, filesize=11.8 K 2024-11-14T02:46:06,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/c02800042c1444309d95a7c8dd26a69d as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/c02800042c1444309d95a7c8dd26a69d 2024-11-14T02:46:06,558 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/c02800042c1444309d95a7c8dd26a69d, entries=150, sequenceid=133, filesize=11.8 K 2024-11-14T02:46:06,559 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for eed6aa27f807a5ebe70b1551484242a3 in 1651ms, sequenceid=133, compaction requested=true 2024-11-14T02:46:06,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for eed6aa27f807a5ebe70b1551484242a3: 2024-11-14T02:46:06,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:46:06,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-14T02:46:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-14T02:46:06,561 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-14T02:46:06,561 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8050 sec 2024-11-14T02:46:06,563 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.8080 sec 2024-11-14T02:46:06,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T02:46:06,913 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:46:06,914 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:46:06,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-14T02:46:06,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T02:46:06,915 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:46:06,916 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:46:06,916 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:46:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T02:46:07,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T02:46:07,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:46:07,068 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing eed6aa27f807a5ebe70b1551484242a3 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-14T02:46:07,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cd59c6dec4224117995d75b82746076d_eed6aa27f807a5ebe70b1551484242a3 is 50, key is test_row_0/A:col10/1731552365148/Put/seqid=0 2024-11-14T02:46:07,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742345_1521 (size=12304) 2024-11-14T02:46:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:07,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. as already flushing 2024-11-14T02:46:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T02:46:07,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552427237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:07,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552427313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:07,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552427313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:07,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552427313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:07,315 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_2' in eed6aa27f807a5ebe70b1551484242a3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:46:07,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:07,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:07,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:07,316 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_2' in eed6aa27f807a5ebe70b1551484242a3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:46:07,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:07,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552427313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:07,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:07,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:07,316 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_1' in eed6aa27f807a5ebe70b1551484242a3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:46:07,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:07,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:07,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:07,316 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_2' in eed6aa27f807a5ebe70b1551484242a3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:46:07,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:07,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:07,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:07,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552427343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:07,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:07,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:07,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:07,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:07,478 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cd59c6dec4224117995d75b82746076d_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cd59c6dec4224117995d75b82746076d_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:07,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/e235dc44296f4ce98fbb8640725bb447, store: [table=TestAcidGuarantees family=A region=eed6aa27f807a5ebe70b1551484242a3] 2024-11-14T02:46:07,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/e235dc44296f4ce98fbb8640725bb447 is 175, key is test_row_0/A:col10/1731552365148/Put/seqid=0 2024-11-14T02:46:07,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742346_1522 (size=31105) 2024-11-14T02:46:07,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T02:46:07,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:07,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552427554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:07,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:07,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:07,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:07,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:07,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552427863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:07,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:07,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:07,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:07,883 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=152, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/e235dc44296f4ce98fbb8640725bb447 2024-11-14T02:46:07,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/e5f22f048bd144fcaa11eafda5ea6c71 is 50, key is test_row_0/B:col10/1731552365148/Put/seqid=0 2024-11-14T02:46:07,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742347_1523 (size=12151) 2024-11-14T02:46:08,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T02:46:08,292 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/e5f22f048bd144fcaa11eafda5ea6c71 2024-11-14T02:46:08,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/2613c694ae68471dbaf1b2c7fc982909 is 50, key is test_row_0/C:col10/1731552365148/Put/seqid=0 2024-11-14T02:46:08,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742348_1524 (size=12151) 2024-11-14T02:46:08,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552428373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:08,732 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/2613c694ae68471dbaf1b2c7fc982909 2024-11-14T02:46:08,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/e235dc44296f4ce98fbb8640725bb447 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/e235dc44296f4ce98fbb8640725bb447 2024-11-14T02:46:08,739 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/e235dc44296f4ce98fbb8640725bb447, entries=150, sequenceid=152, filesize=30.4 K 2024-11-14T02:46:08,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/e5f22f048bd144fcaa11eafda5ea6c71 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/e5f22f048bd144fcaa11eafda5ea6c71 2024-11-14T02:46:08,743 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/e5f22f048bd144fcaa11eafda5ea6c71, entries=150, sequenceid=152, filesize=11.9 K 2024-11-14T02:46:08,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/2613c694ae68471dbaf1b2c7fc982909 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/2613c694ae68471dbaf1b2c7fc982909 2024-11-14T02:46:08,747 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/2613c694ae68471dbaf1b2c7fc982909, entries=150, sequenceid=152, filesize=11.9 K 2024-11-14T02:46:08,748 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for eed6aa27f807a5ebe70b1551484242a3 in 1680ms, sequenceid=152, compaction requested=true 2024-11-14T02:46:08,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for eed6aa27f807a5ebe70b1551484242a3: 2024-11-14T02:46:08,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:46:08,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-14T02:46:08,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-14T02:46:08,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-14T02:46:08,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8330 sec 2024-11-14T02:46:08,752 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.8370 sec 2024-11-14T02:46:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T02:46:09,064 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:46:09,064 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:46:09,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-14T02:46:09,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T02:46:09,066 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:46:09,066 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:46:09,066 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:46:09,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T02:46:09,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-14T02:46:09,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:46:09,218 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing eed6aa27f807a5ebe70b1551484242a3 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-14T02:46:09,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144c4b0e1c1b004a6da94fc4d74e8fb112_eed6aa27f807a5ebe70b1551484242a3 is 50, key is test_row_0/A:col10/1731552367225/Put/seqid=0 2024-11-14T02:46:09,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742349_1525 (size=12304) 2024-11-14T02:46:09,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:09,229 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144c4b0e1c1b004a6da94fc4d74e8fb112_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144c4b0e1c1b004a6da94fc4d74e8fb112_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:09,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/10dd59f461924e3c80134bccb1c5b1b9, store: [table=TestAcidGuarantees family=A region=eed6aa27f807a5ebe70b1551484242a3] 2024-11-14T02:46:09,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/10dd59f461924e3c80134bccb1c5b1b9 is 175, key is test_row_0/A:col10/1731552367225/Put/seqid=0 2024-11-14T02:46:09,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742350_1526 (size=31105) 2024-11-14T02:46:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:09,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. as already flushing 2024-11-14T02:46:09,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552429355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:09,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552429356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:09,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552429357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:09,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552429359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:09,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T02:46:09,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552429393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:09,396 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_2' in eed6aa27f807a5ebe70b1551484242a3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:46:09,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552429463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:09,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:09,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552429463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:09,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552429473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:09,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:09,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552429473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:09,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,634 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/10dd59f461924e3c80134bccb1c5b1b9 2024-11-14T02:46:09,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/345e4d176c554da1bae2a8bb7af868df is 50, key is test_row_0/B:col10/1731552367225/Put/seqid=0 2024-11-14T02:46:09,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742351_1527 (size=12151) 2024-11-14T02:46:09,647 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/345e4d176c554da1bae2a8bb7af868df 2024-11-14T02:46:09,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/608405f96d874902b1a683df4b712ad1 is 50, key is test_row_0/C:col10/1731552367225/Put/seqid=0 2024-11-14T02:46:09,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742352_1528 (size=12151) 2024-11-14T02:46:09,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552429673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:09,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:09,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552429673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:09,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552429683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:09,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:09,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552429683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:09,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T02:46:09,750 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T02:46:09,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:09,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552429984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:09,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:09,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552429984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:09,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:09,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552429993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:09,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:09,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552429993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:09,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:10,061 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/608405f96d874902b1a683df4b712ad1 2024-11-14T02:46:10,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/10dd59f461924e3c80134bccb1c5b1b9 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/10dd59f461924e3c80134bccb1c5b1b9 2024-11-14T02:46:10,066 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/10dd59f461924e3c80134bccb1c5b1b9, entries=150, sequenceid=169, filesize=30.4 K 2024-11-14T02:46:10,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/345e4d176c554da1bae2a8bb7af868df as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/345e4d176c554da1bae2a8bb7af868df 2024-11-14T02:46:10,070 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/345e4d176c554da1bae2a8bb7af868df, entries=150, sequenceid=169, filesize=11.9 K 2024-11-14T02:46:10,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/608405f96d874902b1a683df4b712ad1 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/608405f96d874902b1a683df4b712ad1 2024-11-14T02:46:10,073 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/608405f96d874902b1a683df4b712ad1, entries=150, sequenceid=169, filesize=11.9 K 2024-11-14T02:46:10,073 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for eed6aa27f807a5ebe70b1551484242a3 in 855ms, sequenceid=169, compaction requested=true 2024-11-14T02:46:10,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for eed6aa27f807a5ebe70b1551484242a3: 2024-11-14T02:46:10,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:46:10,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-14T02:46:10,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-14T02:46:10,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-14T02:46:10,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0080 sec 2024-11-14T02:46:10,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.0120 sec 2024-11-14T02:46:10,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T02:46:10,204 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:46:10,204 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:46:10,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-14T02:46:10,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T02:46:10,206 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:46:10,206 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:46:10,206 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:46:10,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T02:46:10,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T02:46:10,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:46:10,358 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing eed6aa27f807a5ebe70b1551484242a3 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-14T02:46:10,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a6c1cbcaa0714f5889ff8dcaf6db290c_eed6aa27f807a5ebe70b1551484242a3 is 50, key is test_row_0/A:col10/1731552369353/Put/seqid=0 2024-11-14T02:46:10,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742353_1529 (size=12304) 2024-11-14T02:46:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:10,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. as already flushing 2024-11-14T02:46:10,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:10,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552430515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:10,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:10,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552430516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:10,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:10,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:10,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:10,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552430517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:10,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T02:46:10,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:10,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552430522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:10,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:10,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:10,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552430623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:10,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552430633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:10,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:10,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552430633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:10,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:10,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552430634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:10,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:10,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,770 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a6c1cbcaa0714f5889ff8dcaf6db290c_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a6c1cbcaa0714f5889ff8dcaf6db290c_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:10,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/34f61aba629b4d20af1859a6f5a3b2ab, store: [table=TestAcidGuarantees family=A region=eed6aa27f807a5ebe70b1551484242a3] 2024-11-14T02:46:10,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/34f61aba629b4d20af1859a6f5a3b2ab is 175, key is test_row_0/A:col10/1731552369353/Put/seqid=0 2024-11-14T02:46:10,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742354_1530 (size=31105) 2024-11-14T02:46:10,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T02:46:10,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:10,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552430833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:10,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:10,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552430843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:10,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:10,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552430843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:10,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552430843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:11,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552431144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:11,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:11,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552431153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:11,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552431153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:11,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552431153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:11,174 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=188, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/34f61aba629b4d20af1859a6f5a3b2ab 2024-11-14T02:46:11,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/458a30af6f624681b212521a7f20e5a3 is 50, key is test_row_0/B:col10/1731552369353/Put/seqid=0 2024-11-14T02:46:11,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742355_1531 (size=12151) 2024-11-14T02:46:11,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T02:46:11,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:11,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552431423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:11,428 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_2' in eed6aa27f807a5ebe70b1551484242a3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:46:11,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:11,583 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/458a30af6f624681b212521a7f20e5a3 2024-11-14T02:46:11,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/9d3a6501d3a1429a9c48caf57eb4726a is 50, key is test_row_0/C:col10/1731552369353/Put/seqid=0 2024-11-14T02:46:11,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742356_1532 (size=12151) 2024-11-14T02:46:11,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:11,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552431653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:11,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:11,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:11,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552431663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:11,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:11,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:11,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552431663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:11,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:11,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:11,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552431663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:11,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:11,995 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/9d3a6501d3a1429a9c48caf57eb4726a 2024-11-14T02:46:11,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/34f61aba629b4d20af1859a6f5a3b2ab as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/34f61aba629b4d20af1859a6f5a3b2ab 2024-11-14T02:46:12,001 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/34f61aba629b4d20af1859a6f5a3b2ab, entries=150, sequenceid=188, filesize=30.4 K 2024-11-14T02:46:12,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/458a30af6f624681b212521a7f20e5a3 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/458a30af6f624681b212521a7f20e5a3 2024-11-14T02:46:12,005 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/458a30af6f624681b212521a7f20e5a3, entries=150, sequenceid=188, filesize=11.9 K 2024-11-14T02:46:12,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/9d3a6501d3a1429a9c48caf57eb4726a as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/9d3a6501d3a1429a9c48caf57eb4726a 2024-11-14T02:46:12,008 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/9d3a6501d3a1429a9c48caf57eb4726a, entries=150, sequenceid=188, filesize=11.9 K 2024-11-14T02:46:12,008 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for eed6aa27f807a5ebe70b1551484242a3 in 1650ms, sequenceid=188, compaction requested=true 2024-11-14T02:46:12,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for eed6aa27f807a5ebe70b1551484242a3: 2024-11-14T02:46:12,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:46:12,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-14T02:46:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-14T02:46:12,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-14T02:46:12,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8030 sec 2024-11-14T02:46:12,012 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.8070 sec 2024-11-14T02:46:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T02:46:12,353 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:46:12,354 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:46:12,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-14T02:46:12,355 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:46:12,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T02:46:12,356 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:46:12,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:46:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T02:46:12,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-14T02:46:12,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:46:12,510 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing eed6aa27f807a5ebe70b1551484242a3 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-14T02:46:12,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a196eb88229245bdaa17c59c8fc800e9_eed6aa27f807a5ebe70b1551484242a3 is 50, key is test_row_0/A:col10/1731552370521/Put/seqid=0 2024-11-14T02:46:12,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742357_1533 (size=12304) 2024-11-14T02:46:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:12,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. as already flushing 2024-11-14T02:46:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T02:46:12,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:12,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552432702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:12,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:12,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:12,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552432703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:12,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:12,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:12,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:12,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:12,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552432706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:12,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:12,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:12,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:12,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:12,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552432708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:12,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:12,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:12,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:12,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:12,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552432813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:12,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:12,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:12,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552432813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:12,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:12,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:12,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:12,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:12,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:12,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:12,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552432823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:12,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:12,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:12,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552432823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:12,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:12,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:12,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:12,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:12,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:12,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:12,927 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a196eb88229245bdaa17c59c8fc800e9_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a196eb88229245bdaa17c59c8fc800e9_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:12,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/c1487fbfd6404b269ea24086ce733afb, store: [table=TestAcidGuarantees family=A region=eed6aa27f807a5ebe70b1551484242a3] 2024-11-14T02:46:12,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/c1487fbfd6404b269ea24086ce733afb is 175, key is test_row_0/A:col10/1731552370521/Put/seqid=0 2024-11-14T02:46:12,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742358_1534 (size=31105) 2024-11-14T02:46:12,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T02:46:13,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:13,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552433023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:13,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:13,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552433023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:13,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:13,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552433033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:13,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:13,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552433033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:13,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:13,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552433333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:13,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:13,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552433333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:13,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:13,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552433343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:13,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:13,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552433343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:13,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,358 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=205, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/c1487fbfd6404b269ea24086ce733afb 2024-11-14T02:46:13,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/0b400f8d4a864029841254b1aa2ef79a is 50, key is test_row_0/B:col10/1731552370521/Put/seqid=0 2024-11-14T02:46:13,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742359_1535 (size=12151) 2024-11-14T02:46:13,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T02:46:13,766 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/0b400f8d4a864029841254b1aa2ef79a 2024-11-14T02:46:13,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/e8466b98fb4a4bae9d7bd8e734b0a2ed is 50, key is test_row_0/C:col10/1731552370521/Put/seqid=0 2024-11-14T02:46:13,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742360_1536 (size=12151) 2024-11-14T02:46:13,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:13,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552433843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:13,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552433843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:13,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552433853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:13,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:13,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552433864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:14,175 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/e8466b98fb4a4bae9d7bd8e734b0a2ed 2024-11-14T02:46:14,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/c1487fbfd6404b269ea24086ce733afb as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/c1487fbfd6404b269ea24086ce733afb 2024-11-14T02:46:14,181 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/c1487fbfd6404b269ea24086ce733afb, entries=150, sequenceid=205, filesize=30.4 K 2024-11-14T02:46:14,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/0b400f8d4a864029841254b1aa2ef79a as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/0b400f8d4a864029841254b1aa2ef79a 2024-11-14T02:46:14,185 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/0b400f8d4a864029841254b1aa2ef79a, entries=150, sequenceid=205, filesize=11.9 K 2024-11-14T02:46:14,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/e8466b98fb4a4bae9d7bd8e734b0a2ed as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/e8466b98fb4a4bae9d7bd8e734b0a2ed 2024-11-14T02:46:14,188 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/e8466b98fb4a4bae9d7bd8e734b0a2ed, entries=150, sequenceid=205, filesize=11.9 K 2024-11-14T02:46:14,189 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for eed6aa27f807a5ebe70b1551484242a3 in 1679ms, sequenceid=205, compaction requested=true 2024-11-14T02:46:14,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for eed6aa27f807a5ebe70b1551484242a3: 2024-11-14T02:46:14,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:46:14,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-14T02:46:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-14T02:46:14,192 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-14T02:46:14,192 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8350 sec 2024-11-14T02:46:14,194 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.8390 sec 2024-11-14T02:46:14,243 DEBUG [regionserver/83c5a5c119d5:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.46153846153846156, tune throughput to 73.08 MB/second 2024-11-14T02:46:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T02:46:14,503 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:46:14,504 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:46:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-14T02:46:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-14T02:46:14,505 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:46:14,506 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:46:14,506 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:46:14,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-14T02:46:14,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-14T02:46:14,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:46:14,658 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing eed6aa27f807a5ebe70b1551484242a3 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-14T02:46:14,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111458a86e450ccc422f95b912fa92c7c5ec_eed6aa27f807a5ebe70b1551484242a3 is 50, key is test_row_0/A:col10/1731552372704/Put/seqid=0 2024-11-14T02:46:14,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742361_1537 (size=12304) 2024-11-14T02:46:14,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-14T02:46:14,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:14,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. as already flushing 2024-11-14T02:46:14,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552434888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:14,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:14,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:14,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:14,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552434889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:14,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:14,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:14,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:14,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552434890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:14,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552434890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:15,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552435003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:15,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552435003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552435004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:15,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552435004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:15,073 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111458a86e450ccc422f95b912fa92c7c5ec_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111458a86e450ccc422f95b912fa92c7c5ec_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:15,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/ad82f9bcfb5940eb9308a4a883ffd270, store: [table=TestAcidGuarantees family=A region=eed6aa27f807a5ebe70b1551484242a3] 2024-11-14T02:46:15,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/ad82f9bcfb5940eb9308a4a883ffd270 is 175, key is test_row_0/A:col10/1731552372704/Put/seqid=0 2024-11-14T02:46:15,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742362_1538 (size=31105) 2024-11-14T02:46:15,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-14T02:46:15,153 INFO [Thread-2222 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2222. 2024-11-14T02:46:15,153 DEBUG [Thread-2222 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:46:15,153 DEBUG [Thread-2222 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:15,153 DEBUG [Thread-2222 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:15,153 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:46:15,153 INFO [Thread-2220 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2220. 2024-11-14T02:46:15,153 DEBUG [Thread-2220 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:46:15,153 DEBUG [Thread-2220 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:15,154 DEBUG [Thread-2220 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:15,154 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:46:15,155 INFO [Thread-2218 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2218. 2024-11-14T02:46:15,155 DEBUG [Thread-2218 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:46:15,155 DEBUG [Thread-2218 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:15,155 DEBUG [Thread-2218 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:15,155 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:46:15,158 INFO [Thread-2216 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2216. 2024-11-14T02:46:15,158 DEBUG [Thread-2216 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:46:15,158 DEBUG [Thread-2216 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:15,158 DEBUG [Thread-2216 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:15,158 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:46:15,159 INFO [Thread-2224 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2224. 2024-11-14T02:46:15,159 DEBUG [Thread-2224 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:46:15,159 DEBUG [Thread-2224 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:15,159 DEBUG [Thread-2224 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:15,159 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:46:15,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:15,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:15,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552435214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552435214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552435214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:15,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552435214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46766 deadline: 1731552435444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:15,445 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 83c5a5c119d5,41455,1731552191259 for 'test_row_2' in eed6aa27f807a5ebe70b1551484242a3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-14T02:46:15,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,478 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/ad82f9bcfb5940eb9308a4a883ffd270 2024-11-14T02:46:15,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/40234a4729854f35b001c004239624f6 is 50, key is test_row_0/B:col10/1731552372704/Put/seqid=0 2024-11-14T02:46:15,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742363_1539 (size=12151) 2024-11-14T02:46:15,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:15,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552435524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552435524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:15,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:15,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552435525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:15,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552435525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-14T02:46:15,895 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/40234a4729854f35b001c004239624f6 2024-11-14T02:46:15,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/4a807fe1ee364240b361a2e2ea3ed1bb is 50, key is test_row_0/C:col10/1731552372704/Put/seqid=0 2024-11-14T02:46:15,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742364_1540 (size=12151) 2024-11-14T02:46:16,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46860 deadline: 1731552436034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:16,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:16,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46804 deadline: 1731552436035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:16,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-14T02:46:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1731552436035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46782 deadline: 1731552436035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., hostname=83c5a5c119d5,41455,1731552191259, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:16,313 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/4a807fe1ee364240b361a2e2ea3ed1bb 2024-11-14T02:46:16,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/ad82f9bcfb5940eb9308a4a883ffd270 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/ad82f9bcfb5940eb9308a4a883ffd270 2024-11-14T02:46:16,330 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/ad82f9bcfb5940eb9308a4a883ffd270, entries=150, sequenceid=224, filesize=30.4 K 2024-11-14T02:46:16,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/40234a4729854f35b001c004239624f6 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/40234a4729854f35b001c004239624f6 2024-11-14T02:46:16,335 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/40234a4729854f35b001c004239624f6, entries=150, sequenceid=224, filesize=11.9 K 2024-11-14T02:46:16,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/4a807fe1ee364240b361a2e2ea3ed1bb as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/4a807fe1ee364240b361a2e2ea3ed1bb 2024-11-14T02:46:16,340 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/4a807fe1ee364240b361a2e2ea3ed1bb, entries=150, sequenceid=224, filesize=11.9 K 2024-11-14T02:46:16,341 INFO [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for eed6aa27f807a5ebe70b1551484242a3 in 1683ms, sequenceid=224, compaction requested=true 2024-11-14T02:46:16,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for eed6aa27f807a5ebe70b1551484242a3: 2024-11-14T02:46:16,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:46:16,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/83c5a5c119d5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-14T02:46:16,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-14T02:46:16,346 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-14T02:46:16,346 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8370 sec 2024-11-14T02:46:16,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.8420 sec 2024-11-14T02:46:16,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-14T02:46:16,665 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:46:17,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41455 {}] regionserver.HRegion(8855): Flush requested on eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:17,050 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T02:46:17,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eed6aa27f807a5ebe70b1551484242a3 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-14T02:46:17,050 INFO [Thread-2207 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2207. 2024-11-14T02:46:17,050 INFO [Thread-2213 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2213. 2024-11-14T02:46:17,050 INFO [Thread-2205 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2205. 2024-11-14T02:46:17,050 DEBUG [Thread-2205 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:46:17,050 DEBUG [Thread-2213 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:46:17,050 DEBUG [Thread-2207 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:46:17,050 DEBUG [Thread-2205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:17,050 DEBUG [Thread-2213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:17,050 DEBUG [Thread-2207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:17,050 DEBUG [Thread-2205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:17,050 DEBUG [Thread-2213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:17,050 DEBUG [Thread-2207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:17,050 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:46:17,050 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:46:17,051 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:46:17,055 INFO [Thread-2211 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2211. 2024-11-14T02:46:17,055 DEBUG [Thread-2211 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:46:17,055 DEBUG [Thread-2211 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:17,055 DEBUG [Thread-2211 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:17,055 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:46:17,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143937805a01284f86b8e170d7bda17499_eed6aa27f807a5ebe70b1551484242a3 is 50, key is test_row_0/A:col10/1731552377046/Put/seqid=0 2024-11-14T02:46:17,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742365_1541 (size=12304) 2024-11-14T02:46:17,461 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:17,468 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143937805a01284f86b8e170d7bda17499_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143937805a01284f86b8e170d7bda17499_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:17,469 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/2c4cf4a2620f4909b4eaf6e713f831af, store: [table=TestAcidGuarantees family=A region=eed6aa27f807a5ebe70b1551484242a3] 2024-11-14T02:46:17,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/2c4cf4a2620f4909b4eaf6e713f831af is 175, key is test_row_0/A:col10/1731552377046/Put/seqid=0 2024-11-14T02:46:17,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742366_1542 (size=31105) 2024-11-14T02:46:17,876 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/2c4cf4a2620f4909b4eaf6e713f831af 2024-11-14T02:46:17,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/48a3c9cf0ca54adeae5580da3834b576 is 50, key is test_row_0/B:col10/1731552377046/Put/seqid=0 2024-11-14T02:46:17,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742367_1543 (size=12151) 2024-11-14T02:46:18,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/48a3c9cf0ca54adeae5580da3834b576 2024-11-14T02:46:18,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/34c0db53324741158a9c6a2093df4b83 is 50, key is test_row_0/C:col10/1731552377046/Put/seqid=0 2024-11-14T02:46:18,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742368_1544 (size=12151) 2024-11-14T02:46:18,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/34c0db53324741158a9c6a2093df4b83 2024-11-14T02:46:18,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/2c4cf4a2620f4909b4eaf6e713f831af as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/2c4cf4a2620f4909b4eaf6e713f831af 2024-11-14T02:46:18,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/2c4cf4a2620f4909b4eaf6e713f831af, entries=150, sequenceid=244, filesize=30.4 K 2024-11-14T02:46:18,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/48a3c9cf0ca54adeae5580da3834b576 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/48a3c9cf0ca54adeae5580da3834b576 2024-11-14T02:46:18,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/48a3c9cf0ca54adeae5580da3834b576, entries=150, sequenceid=244, filesize=11.9 K 2024-11-14T02:46:18,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/34c0db53324741158a9c6a2093df4b83 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/34c0db53324741158a9c6a2093df4b83 2024-11-14T02:46:18,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/34c0db53324741158a9c6a2093df4b83, entries=150, sequenceid=244, filesize=11.9 K 2024-11-14T02:46:18,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=6.71 KB/6870 for eed6aa27f807a5ebe70b1551484242a3 in 1704ms, sequenceid=244, compaction requested=true 2024-11-14T02:46:18,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eed6aa27f807a5ebe70b1551484242a3: 2024-11-14T02:46:18,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eed6aa27f807a5ebe70b1551484242a3:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:46:18,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:18,754 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 11 store files, 0 compacting, 11 eligible, 16 blocking 2024-11-14T02:46:18,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eed6aa27f807a5ebe70b1551484242a3:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:46:18,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:18,754 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 11 store files, 0 compacting, 11 eligible, 16 blocking 2024-11-14T02:46:18,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eed6aa27f807a5ebe70b1551484242a3:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:46:18,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:46:18,758 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 10 files of size 310503 starting at candidate #0 after considering 44 permutations with 44 in ratio 2024-11-14T02:46:18,758 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1541): eed6aa27f807a5ebe70b1551484242a3/A is initiating minor compaction 2024-11-14T02:46:18,758 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 10 files of size 120963 starting at candidate #0 after considering 44 permutations with 44 in ratio 2024-11-14T02:46:18,758 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eed6aa27f807a5ebe70b1551484242a3/A in TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:46:18,758 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): eed6aa27f807a5ebe70b1551484242a3/B is initiating minor compaction 2024-11-14T02:46:18,758 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eed6aa27f807a5ebe70b1551484242a3/B in TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:46:18,758 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/b209a940ce75415ca6ad0714bdbb28ea, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/a165b5285f5a45528bca3cc7e1e242a0, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/5cfee9372ef6490687b37948288912f3, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/4a55b7b265934843afc883a28e8fdbee, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/ffecef329e374a29b5f7270d4e7572b8, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/e235dc44296f4ce98fbb8640725bb447, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/10dd59f461924e3c80134bccb1c5b1b9, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/34f61aba629b4d20af1859a6f5a3b2ab, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/c1487fbfd6404b269ea24086ce733afb, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/ad82f9bcfb5940eb9308a4a883ffd270] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp, totalSize=303.2 K 2024-11-14T02:46:18,758 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/a9fd589f8a744d67a2eac84d25278810, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/2d1bf29696cf40f8acfb601f44e829b0, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/6234968d05e848debb344d17f8313299, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/c9f24f0658be47c08ebe98b3dfb405e2, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/15ae4822a1dd490daa91fca227d5459a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/e5f22f048bd144fcaa11eafda5ea6c71, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/345e4d176c554da1bae2a8bb7af868df, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/458a30af6f624681b212521a7f20e5a3, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/0b400f8d4a864029841254b1aa2ef79a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/40234a4729854f35b001c004239624f6] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp, totalSize=118.1 K 2024-11-14T02:46:18,758 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=false priority=5 throughput controller=DefaultCompactionThroughputController [maxThroughput=73.08 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:46:18,758 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting a9fd589f8a744d67a2eac84d25278810, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56 2024-11-14T02:46:18,758 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. files: [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/b209a940ce75415ca6ad0714bdbb28ea, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/a165b5285f5a45528bca3cc7e1e242a0, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/5cfee9372ef6490687b37948288912f3, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/4a55b7b265934843afc883a28e8fdbee, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/ffecef329e374a29b5f7270d4e7572b8, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/e235dc44296f4ce98fbb8640725bb447, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/10dd59f461924e3c80134bccb1c5b1b9, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/34f61aba629b4d20af1859a6f5a3b2ab, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/c1487fbfd6404b269ea24086ce733afb, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/ad82f9bcfb5940eb9308a4a883ffd270] 2024-11-14T02:46:18,758 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d1bf29696cf40f8acfb601f44e829b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78 2024-11-14T02:46:18,758 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting b209a940ce75415ca6ad0714bdbb28ea, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56 2024-11-14T02:46:18,758 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 6234968d05e848debb344d17f8313299, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96 2024-11-14T02:46:18,758 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting a165b5285f5a45528bca3cc7e1e242a0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78 2024-11-14T02:46:18,758 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting c9f24f0658be47c08ebe98b3dfb405e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116 2024-11-14T02:46:18,758 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cfee9372ef6490687b37948288912f3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=96 2024-11-14T02:46:18,758 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 15ae4822a1dd490daa91fca227d5459a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133 2024-11-14T02:46:18,758 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a55b7b265934843afc883a28e8fdbee, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116 2024-11-14T02:46:18,758 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting e5f22f048bd144fcaa11eafda5ea6c71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152 2024-11-14T02:46:18,758 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffecef329e374a29b5f7270d4e7572b8, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133 2024-11-14T02:46:18,758 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 345e4d176c554da1bae2a8bb7af868df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169 2024-11-14T02:46:18,758 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting e235dc44296f4ce98fbb8640725bb447, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=152 2024-11-14T02:46:18,758 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 458a30af6f624681b212521a7f20e5a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188 2024-11-14T02:46:18,759 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10dd59f461924e3c80134bccb1c5b1b9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=169 2024-11-14T02:46:18,759 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b400f8d4a864029841254b1aa2ef79a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205 2024-11-14T02:46:18,759 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 40234a4729854f35b001c004239624f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224 2024-11-14T02:46:18,759 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34f61aba629b4d20af1859a6f5a3b2ab, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=188 2024-11-14T02:46:18,759 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1487fbfd6404b269ea24086ce733afb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=205 2024-11-14T02:46:18,759 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad82f9bcfb5940eb9308a4a883ffd270, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=224 2024-11-14T02:46:18,770 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=false store=[table=TestAcidGuarantees family=A region=eed6aa27f807a5ebe70b1551484242a3] 2024-11-14T02:46:18,771 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114e1ccfcbdd4e442309c936ab80f613922_eed6aa27f807a5ebe70b1551484242a3 store=[table=TestAcidGuarantees family=A region=eed6aa27f807a5ebe70b1551484242a3] 2024-11-14T02:46:18,773 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eed6aa27f807a5ebe70b1551484242a3#B#compaction#458 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 73.08 MB/second 2024-11-14T02:46:18,773 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/86316b3803c64246b1ee603888c8402d is 50, key is test_row_0/B:col10/1731552372704/Put/seqid=0 2024-11-14T02:46:18,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742369_1545 (size=12595) 2024-11-14T02:46:18,777 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=false file=d41d8cd98f00b204e9800998ecf8427e20241114e1ccfcbdd4e442309c936ab80f613922_eed6aa27f807a5ebe70b1551484242a3, store=[table=TestAcidGuarantees family=A region=eed6aa27f807a5ebe70b1551484242a3] 2024-11-14T02:46:18,777 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e1ccfcbdd4e442309c936ab80f613922_eed6aa27f807a5ebe70b1551484242a3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=eed6aa27f807a5ebe70b1551484242a3] 2024-11-14T02:46:18,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742370_1546 (size=4469) 2024-11-14T02:46:19,183 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eed6aa27f807a5ebe70b1551484242a3#A#compaction#459 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 73.08 MB/second 2024-11-14T02:46:19,184 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/0b1596705b3648088ae1bf69c7d5f05b is 175, key is test_row_0/A:col10/1731552372704/Put/seqid=0 2024-11-14T02:46:19,187 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/86316b3803c64246b1ee603888c8402d as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/86316b3803c64246b1ee603888c8402d 2024-11-14T02:46:19,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742371_1547 (size=31549) 2024-11-14T02:46:19,192 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 10 file(s) in eed6aa27f807a5ebe70b1551484242a3/B of eed6aa27f807a5ebe70b1551484242a3 into 86316b3803c64246b1ee603888c8402d(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:19,192 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eed6aa27f807a5ebe70b1551484242a3: 2024-11-14T02:46:19,193 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., storeName=eed6aa27f807a5ebe70b1551484242a3/B, priority=5, startTime=1731552378754; duration=0sec 2024-11-14T02:46:19,193 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:46:19,193 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eed6aa27f807a5ebe70b1551484242a3:B 2024-11-14T02:46:19,193 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 11 store files, 0 compacting, 11 eligible, 16 blocking 2024-11-14T02:46:19,195 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 10 files of size 120963 starting at candidate #0 after considering 44 permutations with 44 in ratio 2024-11-14T02:46:19,195 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1541): eed6aa27f807a5ebe70b1551484242a3/C is initiating minor compaction 2024-11-14T02:46:19,195 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eed6aa27f807a5ebe70b1551484242a3/C in TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:46:19,196 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/f367540fdfcd4b0da2d7d98a71e0ad97, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/bdc5f0f05ca149ba84014e7ec40d03e8, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/05abe55e8cbf45e9955881378940f3e5, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/f6924a074fa04038b402970113b15592, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/c02800042c1444309d95a7c8dd26a69d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/2613c694ae68471dbaf1b2c7fc982909, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/608405f96d874902b1a683df4b712ad1, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/9d3a6501d3a1429a9c48caf57eb4726a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/e8466b98fb4a4bae9d7bd8e734b0a2ed, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/4a807fe1ee364240b361a2e2ea3ed1bb] into tmpdir=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp, totalSize=118.1 K 2024-11-14T02:46:19,196 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting f367540fdfcd4b0da2d7d98a71e0ad97, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56 2024-11-14T02:46:19,196 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting bdc5f0f05ca149ba84014e7ec40d03e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78 2024-11-14T02:46:19,196 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 05abe55e8cbf45e9955881378940f3e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96 2024-11-14T02:46:19,196 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting f6924a074fa04038b402970113b15592, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116 2024-11-14T02:46:19,196 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting c02800042c1444309d95a7c8dd26a69d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133 2024-11-14T02:46:19,196 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 2613c694ae68471dbaf1b2c7fc982909, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152 2024-11-14T02:46:19,196 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 608405f96d874902b1a683df4b712ad1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169 2024-11-14T02:46:19,196 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d3a6501d3a1429a9c48caf57eb4726a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188 2024-11-14T02:46:19,196 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting e8466b98fb4a4bae9d7bd8e734b0a2ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205 2024-11-14T02:46:19,196 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a807fe1ee364240b361a2e2ea3ed1bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224 2024-11-14T02:46:19,207 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eed6aa27f807a5ebe70b1551484242a3#C#compaction#460 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 73.08 MB/second 2024-11-14T02:46:19,207 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/40751064f1164a8498e615b046ab3317 is 50, key is test_row_0/C:col10/1731552372704/Put/seqid=0 2024-11-14T02:46:19,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742372_1548 (size=12595) 2024-11-14T02:46:19,602 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/0b1596705b3648088ae1bf69c7d5f05b as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/0b1596705b3648088ae1bf69c7d5f05b 2024-11-14T02:46:19,608 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 10 file(s) in eed6aa27f807a5ebe70b1551484242a3/A of eed6aa27f807a5ebe70b1551484242a3 into 0b1596705b3648088ae1bf69c7d5f05b(size=30.8 K), total size for store is 61.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:19,608 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eed6aa27f807a5ebe70b1551484242a3: 2024-11-14T02:46:19,608 INFO [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., storeName=eed6aa27f807a5ebe70b1551484242a3/A, priority=5, startTime=1731552378754; duration=0sec 2024-11-14T02:46:19,608 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:19,608 DEBUG [RS:0;83c5a5c119d5:41455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eed6aa27f807a5ebe70b1551484242a3:A 2024-11-14T02:46:19,615 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/40751064f1164a8498e615b046ab3317 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/40751064f1164a8498e615b046ab3317 2024-11-14T02:46:19,621 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 10 file(s) in eed6aa27f807a5ebe70b1551484242a3/C of eed6aa27f807a5ebe70b1551484242a3 into 40751064f1164a8498e615b046ab3317(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:19,621 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eed6aa27f807a5ebe70b1551484242a3: 2024-11-14T02:46:19,621 INFO [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., storeName=eed6aa27f807a5ebe70b1551484242a3/C, priority=5, startTime=1731552378754; duration=0sec 2024-11-14T02:46:19,621 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:19,621 DEBUG [RS:0;83c5a5c119d5:41455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eed6aa27f807a5ebe70b1551484242a3:C 2024-11-14T02:46:25,467 INFO [Thread-2209 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2209. 2024-11-14T02:46:25,468 DEBUG [Thread-2209 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:46:25,468 DEBUG [Thread-2209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:25,468 DEBUG [Thread-2209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:25,468 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:46:25,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T02:46:25,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-14T02:46:25,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-14T02:46:25,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-14T02:46:25,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-14T02:46:25,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-14T02:46:25,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T02:46:25,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3444 2024-11-14T02:46:25,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3506 2024-11-14T02:46:25,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3374 2024-11-14T02:46:25,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3430 2024-11-14T02:46:25,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3492 2024-11-14T02:46:25,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T02:46:25,470 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T02:46:25,470 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T02:46:25,470 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:25,470 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:25,470 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:46:25,472 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T02:46:25,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T02:46:25,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-14T02:46:25,476 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552385476"}]},"ts":"1731552385476"} 2024-11-14T02:46:25,478 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T02:46:25,478 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T02:46:25,479 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T02:46:25,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eed6aa27f807a5ebe70b1551484242a3, UNASSIGN}] 2024-11-14T02:46:25,482 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eed6aa27f807a5ebe70b1551484242a3, UNASSIGN 2024-11-14T02:46:25,483 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=eed6aa27f807a5ebe70b1551484242a3, regionState=CLOSING, regionLocation=83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:25,484 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eed6aa27f807a5ebe70b1551484242a3, UNASSIGN because future has completed 2024-11-14T02:46:25,484 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T02:46:25,484 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259}] 2024-11-14T02:46:25,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-14T02:46:25,637 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(122): Close eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:25,638 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T02:46:25,638 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1722): Closing eed6aa27f807a5ebe70b1551484242a3, disabling compactions & flushes 2024-11-14T02:46:25,638 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:46:25,638 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:46:25,638 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. after waiting 0 ms 2024-11-14T02:46:25,638 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:46:25,638 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(2902): Flushing eed6aa27f807a5ebe70b1551484242a3 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-14T02:46:25,651 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ef62fe0284d344d3b4575619e35f54a1_eed6aa27f807a5ebe70b1551484242a3 is 50, key is test_row_1/A:col10/1731552377054/Put/seqid=0 2024-11-14T02:46:25,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742373_1549 (size=9814) 2024-11-14T02:46:25,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-14T02:46:26,056 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:26,067 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ef62fe0284d344d3b4575619e35f54a1_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ef62fe0284d344d3b4575619e35f54a1_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:26,068 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/9b1f82ecdf224ed992e3629e3033d7d7, store: [table=TestAcidGuarantees family=A region=eed6aa27f807a5ebe70b1551484242a3] 2024-11-14T02:46:26,069 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/9b1f82ecdf224ed992e3629e3033d7d7 is 175, key is test_row_1/A:col10/1731552377054/Put/seqid=0 2024-11-14T02:46:26,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742374_1550 (size=22461) 2024-11-14T02:46:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-14T02:46:26,475 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/9b1f82ecdf224ed992e3629e3033d7d7 2024-11-14T02:46:26,487 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/47c56043639943b29cb50a6739b38993 is 50, key is test_row_1/B:col10/1731552377054/Put/seqid=0 2024-11-14T02:46:26,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742375_1551 (size=9757) 2024-11-14T02:46:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-14T02:46:26,892 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/47c56043639943b29cb50a6739b38993 2024-11-14T02:46:26,904 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/6d95263e310f4c67afc80f0ccfaef52b is 50, key is test_row_1/C:col10/1731552377054/Put/seqid=0 2024-11-14T02:46:26,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742376_1552 (size=9757) 2024-11-14T02:46:27,311 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/6d95263e310f4c67afc80f0ccfaef52b 2024-11-14T02:46:27,320 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/A/9b1f82ecdf224ed992e3629e3033d7d7 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/9b1f82ecdf224ed992e3629e3033d7d7 2024-11-14T02:46:27,326 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/9b1f82ecdf224ed992e3629e3033d7d7, entries=100, sequenceid=252, filesize=21.9 K 2024-11-14T02:46:27,327 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/B/47c56043639943b29cb50a6739b38993 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/47c56043639943b29cb50a6739b38993 2024-11-14T02:46:27,330 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/47c56043639943b29cb50a6739b38993, entries=100, sequenceid=252, filesize=9.5 K 2024-11-14T02:46:27,331 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/.tmp/C/6d95263e310f4c67afc80f0ccfaef52b as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/6d95263e310f4c67afc80f0ccfaef52b 2024-11-14T02:46:27,334 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/6d95263e310f4c67afc80f0ccfaef52b, entries=100, sequenceid=252, filesize=9.5 K 2024-11-14T02:46:27,335 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for eed6aa27f807a5ebe70b1551484242a3 in 1697ms, sequenceid=252, compaction requested=true 2024-11-14T02:46:27,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/0b364ce6fcc64b6fab03df0b764e595d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/b3348773285946698a8003a7d908a9a4, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/b209a940ce75415ca6ad0714bdbb28ea, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/9c3d3d12ff364543b6d52a1920df0dbb, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/a165b5285f5a45528bca3cc7e1e242a0, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/5cfee9372ef6490687b37948288912f3, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/4a55b7b265934843afc883a28e8fdbee, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/ffecef329e374a29b5f7270d4e7572b8, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/e235dc44296f4ce98fbb8640725bb447, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/10dd59f461924e3c80134bccb1c5b1b9, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/34f61aba629b4d20af1859a6f5a3b2ab, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/c1487fbfd6404b269ea24086ce733afb, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/ad82f9bcfb5940eb9308a4a883ffd270] to archive 2024-11-14T02:46:27,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:46:27,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/0b364ce6fcc64b6fab03df0b764e595d to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/0b364ce6fcc64b6fab03df0b764e595d 2024-11-14T02:46:27,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/b3348773285946698a8003a7d908a9a4 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/b3348773285946698a8003a7d908a9a4 2024-11-14T02:46:27,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/b209a940ce75415ca6ad0714bdbb28ea to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/b209a940ce75415ca6ad0714bdbb28ea 2024-11-14T02:46:27,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/9c3d3d12ff364543b6d52a1920df0dbb to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/9c3d3d12ff364543b6d52a1920df0dbb 2024-11-14T02:46:27,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/a165b5285f5a45528bca3cc7e1e242a0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/a165b5285f5a45528bca3cc7e1e242a0 2024-11-14T02:46:27,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/5cfee9372ef6490687b37948288912f3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/5cfee9372ef6490687b37948288912f3 2024-11-14T02:46:27,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/4a55b7b265934843afc883a28e8fdbee to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/4a55b7b265934843afc883a28e8fdbee 2024-11-14T02:46:27,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/ffecef329e374a29b5f7270d4e7572b8 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/ffecef329e374a29b5f7270d4e7572b8 2024-11-14T02:46:27,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/e235dc44296f4ce98fbb8640725bb447 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/e235dc44296f4ce98fbb8640725bb447 2024-11-14T02:46:27,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/10dd59f461924e3c80134bccb1c5b1b9 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/10dd59f461924e3c80134bccb1c5b1b9 2024-11-14T02:46:27,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/34f61aba629b4d20af1859a6f5a3b2ab to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/34f61aba629b4d20af1859a6f5a3b2ab 2024-11-14T02:46:27,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/c1487fbfd6404b269ea24086ce733afb to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/c1487fbfd6404b269ea24086ce733afb 2024-11-14T02:46:27,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/ad82f9bcfb5940eb9308a4a883ffd270 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/ad82f9bcfb5940eb9308a4a883ffd270 2024-11-14T02:46:27,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/a8d4db196152456f809396385630d025, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/6b8ede0abbd040c1b7fccc427e73adb9, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/a9fd589f8a744d67a2eac84d25278810, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/fc9ff8dbcafb4dd782e549d0fa4ff9ed, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/2d1bf29696cf40f8acfb601f44e829b0, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/6234968d05e848debb344d17f8313299, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/c9f24f0658be47c08ebe98b3dfb405e2, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/15ae4822a1dd490daa91fca227d5459a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/e5f22f048bd144fcaa11eafda5ea6c71, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/345e4d176c554da1bae2a8bb7af868df, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/458a30af6f624681b212521a7f20e5a3, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/0b400f8d4a864029841254b1aa2ef79a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/40234a4729854f35b001c004239624f6] to archive 2024-11-14T02:46:27,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:46:27,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/a8d4db196152456f809396385630d025 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/a8d4db196152456f809396385630d025 2024-11-14T02:46:27,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/6b8ede0abbd040c1b7fccc427e73adb9 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/6b8ede0abbd040c1b7fccc427e73adb9 2024-11-14T02:46:27,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/a9fd589f8a744d67a2eac84d25278810 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/a9fd589f8a744d67a2eac84d25278810 2024-11-14T02:46:27,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/fc9ff8dbcafb4dd782e549d0fa4ff9ed to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/fc9ff8dbcafb4dd782e549d0fa4ff9ed 2024-11-14T02:46:27,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/2d1bf29696cf40f8acfb601f44e829b0 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/2d1bf29696cf40f8acfb601f44e829b0 2024-11-14T02:46:27,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/6234968d05e848debb344d17f8313299 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/6234968d05e848debb344d17f8313299 2024-11-14T02:46:27,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/c9f24f0658be47c08ebe98b3dfb405e2 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/c9f24f0658be47c08ebe98b3dfb405e2 2024-11-14T02:46:27,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/15ae4822a1dd490daa91fca227d5459a to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/15ae4822a1dd490daa91fca227d5459a 2024-11-14T02:46:27,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/e5f22f048bd144fcaa11eafda5ea6c71 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/e5f22f048bd144fcaa11eafda5ea6c71 2024-11-14T02:46:27,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/345e4d176c554da1bae2a8bb7af868df to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/345e4d176c554da1bae2a8bb7af868df 2024-11-14T02:46:27,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/458a30af6f624681b212521a7f20e5a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/458a30af6f624681b212521a7f20e5a3 2024-11-14T02:46:27,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/0b400f8d4a864029841254b1aa2ef79a to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/0b400f8d4a864029841254b1aa2ef79a 2024-11-14T02:46:27,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/40234a4729854f35b001c004239624f6 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/40234a4729854f35b001c004239624f6 2024-11-14T02:46:27,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/cee87745477244acb23f6d2fd4db9690, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/3100a2f1f0724d6db149cec8199b6a54, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/f367540fdfcd4b0da2d7d98a71e0ad97, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/c6dba4a94b28458690be152211c6958a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/bdc5f0f05ca149ba84014e7ec40d03e8, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/05abe55e8cbf45e9955881378940f3e5, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/f6924a074fa04038b402970113b15592, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/c02800042c1444309d95a7c8dd26a69d, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/2613c694ae68471dbaf1b2c7fc982909, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/608405f96d874902b1a683df4b712ad1, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/9d3a6501d3a1429a9c48caf57eb4726a, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/e8466b98fb4a4bae9d7bd8e734b0a2ed, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/4a807fe1ee364240b361a2e2ea3ed1bb] to archive 2024-11-14T02:46:27,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:46:27,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/cee87745477244acb23f6d2fd4db9690 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/cee87745477244acb23f6d2fd4db9690 2024-11-14T02:46:27,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/3100a2f1f0724d6db149cec8199b6a54 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/3100a2f1f0724d6db149cec8199b6a54 2024-11-14T02:46:27,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/f367540fdfcd4b0da2d7d98a71e0ad97 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/f367540fdfcd4b0da2d7d98a71e0ad97 2024-11-14T02:46:27,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/c6dba4a94b28458690be152211c6958a to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/c6dba4a94b28458690be152211c6958a 2024-11-14T02:46:27,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/bdc5f0f05ca149ba84014e7ec40d03e8 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/bdc5f0f05ca149ba84014e7ec40d03e8 2024-11-14T02:46:27,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/05abe55e8cbf45e9955881378940f3e5 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/05abe55e8cbf45e9955881378940f3e5 2024-11-14T02:46:27,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/f6924a074fa04038b402970113b15592 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/f6924a074fa04038b402970113b15592 2024-11-14T02:46:27,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/c02800042c1444309d95a7c8dd26a69d to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/c02800042c1444309d95a7c8dd26a69d 2024-11-14T02:46:27,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/2613c694ae68471dbaf1b2c7fc982909 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/2613c694ae68471dbaf1b2c7fc982909 2024-11-14T02:46:27,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/608405f96d874902b1a683df4b712ad1 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/608405f96d874902b1a683df4b712ad1 2024-11-14T02:46:27,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/9d3a6501d3a1429a9c48caf57eb4726a to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/9d3a6501d3a1429a9c48caf57eb4726a 2024-11-14T02:46:27,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/e8466b98fb4a4bae9d7bd8e734b0a2ed to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/e8466b98fb4a4bae9d7bd8e734b0a2ed 2024-11-14T02:46:27,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/4a807fe1ee364240b361a2e2ea3ed1bb to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/4a807fe1ee364240b361a2e2ea3ed1bb 2024-11-14T02:46:27,373 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/recovered.edits/255.seqid, newMaxSeqId=255, maxSeqId=4 2024-11-14T02:46:27,373 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. 2024-11-14T02:46:27,373 DEBUG [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1676): Region close journal for eed6aa27f807a5ebe70b1551484242a3: Waiting for close lock at 1731552385638Running coprocessor pre-close hooks at 1731552385638Disabling compacts and flushes for region at 1731552385638Disabling writes for close at 1731552385638Obtaining lock to block concurrent updates at 1731552385638Preparing flush snapshotting stores in eed6aa27f807a5ebe70b1551484242a3 at 1731552385639 (+1 ms)Finished memstore snapshotting TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731552385639Flushing stores of TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3. at 1731552385640 (+1 ms)Flushing eed6aa27f807a5ebe70b1551484242a3/A: creating writer at 1731552385641 (+1 ms)Flushing mob file eed6aa27f807a5ebe70b1551484242a3/A: appending metadata at 1731552385651 (+10 ms)Flushing mob file eed6aa27f807a5ebe70b1551484242a3/A: closing flushed file at 1731552385651Flushing eed6aa27f807a5ebe70b1551484242a3/A: appending metadata at 1731552386068 (+417 ms)Flushing eed6aa27f807a5ebe70b1551484242a3/A: closing flushed file at 1731552386069 (+1 ms)Flushing eed6aa27f807a5ebe70b1551484242a3/B: creating writer at 1731552386484 (+415 ms)Flushing eed6aa27f807a5ebe70b1551484242a3/B: appending metadata at 1731552386486 (+2 ms)Flushing eed6aa27f807a5ebe70b1551484242a3/B: closing flushed file at 1731552386486Flushing eed6aa27f807a5ebe70b1551484242a3/C: creating writer at 1731552386900 (+414 ms)Flushing eed6aa27f807a5ebe70b1551484242a3/C: appending metadata at 1731552386903 (+3 ms)Flushing eed6aa27f807a5ebe70b1551484242a3/C: closing flushed file at 1731552386903Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c1fa95: reopening flushed file at 1731552387319 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72ba9f28: reopening flushed file at 1731552387326 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e4707c8: reopening flushed file at 1731552387330 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for eed6aa27f807a5ebe70b1551484242a3 in 1697ms, sequenceid=252, compaction requested=true at 1731552387335 (+5 ms)Writing region close event to WAL at 1731552387370 (+35 ms)Running coprocessor post-close hooks at 1731552387373 (+3 ms)Closed at 1731552387373 2024-11-14T02:46:27,375 INFO [RS_CLOSE_REGION-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(157): Closed eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:27,375 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=eed6aa27f807a5ebe70b1551484242a3, regionState=CLOSED 2024-11-14T02:46:27,376 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 because future has completed 2024-11-14T02:46:27,378 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-14T02:46:27,378 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseRegionProcedure eed6aa27f807a5ebe70b1551484242a3, server=83c5a5c119d5,41455,1731552191259 in 1.8930 sec 2024-11-14T02:46:27,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=170, resume processing ppid=169 2024-11-14T02:46:27,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, ppid=169, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eed6aa27f807a5ebe70b1551484242a3, UNASSIGN in 1.8970 sec 2024-11-14T02:46:27,381 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-14T02:46:27,381 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9010 sec 2024-11-14T02:46:27,381 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552387381"}]},"ts":"1731552387381"} 2024-11-14T02:46:27,382 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T02:46:27,382 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T02:46:27,384 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9110 sec 2024-11-14T02:46:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-14T02:46:27,625 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:46:27,626 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T02:46:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:46:27,631 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:46:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-14T02:46:27,633 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=172, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:46:27,638 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:27,641 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A, FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B, FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C, FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/recovered.edits] 2024-11-14T02:46:27,645 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/0b1596705b3648088ae1bf69c7d5f05b to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/0b1596705b3648088ae1bf69c7d5f05b 2024-11-14T02:46:27,646 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/2c4cf4a2620f4909b4eaf6e713f831af to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/2c4cf4a2620f4909b4eaf6e713f831af 2024-11-14T02:46:27,648 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/9b1f82ecdf224ed992e3629e3033d7d7 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/A/9b1f82ecdf224ed992e3629e3033d7d7 2024-11-14T02:46:27,651 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/47c56043639943b29cb50a6739b38993 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/47c56043639943b29cb50a6739b38993 2024-11-14T02:46:27,652 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/48a3c9cf0ca54adeae5580da3834b576 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/48a3c9cf0ca54adeae5580da3834b576 2024-11-14T02:46:27,653 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/86316b3803c64246b1ee603888c8402d to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/B/86316b3803c64246b1ee603888c8402d 2024-11-14T02:46:27,656 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/34c0db53324741158a9c6a2093df4b83 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/34c0db53324741158a9c6a2093df4b83 2024-11-14T02:46:27,658 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/40751064f1164a8498e615b046ab3317 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/40751064f1164a8498e615b046ab3317 2024-11-14T02:46:27,659 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/6d95263e310f4c67afc80f0ccfaef52b to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/C/6d95263e310f4c67afc80f0ccfaef52b 2024-11-14T02:46:27,662 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/recovered.edits/255.seqid to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3/recovered.edits/255.seqid 2024-11-14T02:46:27,663 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/default/TestAcidGuarantees/eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:27,663 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T02:46:27,663 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T02:46:27,664 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T02:46:27,668 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143937805a01284f86b8e170d7bda17499_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143937805a01284f86b8e170d7bda17499_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:27,670 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143d182453eb2447beaf8ecad4f390f73b_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143d182453eb2447beaf8ecad4f390f73b_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:27,672 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114429460c4648e483db3b213fcca9598ba_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114429460c4648e483db3b213fcca9598ba_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:27,674 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144c4b0e1c1b004a6da94fc4d74e8fb112_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144c4b0e1c1b004a6da94fc4d74e8fb112_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:27,676 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144d4c2744f77f4489bc2217d17db3edab_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144d4c2744f77f4489bc2217d17db3edab_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:27,678 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111458a86e450ccc422f95b912fa92c7c5ec_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111458a86e450ccc422f95b912fa92c7c5ec_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:27,680 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145ef3a1e938cf41e0b2dc96746aaa12e5_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145ef3a1e938cf41e0b2dc96746aaa12e5_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:27,682 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111469c4bafc403a4549a806b2d1d13df8ec_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111469c4bafc403a4549a806b2d1d13df8ec_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:27,684 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111486cda87a2b22473e83943c4378454e09_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111486cda87a2b22473e83943c4378454e09_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:27,685 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a196eb88229245bdaa17c59c8fc800e9_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a196eb88229245bdaa17c59c8fc800e9_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:27,686 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a3045b68bdd84b87b028e2482c6c55b9_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a3045b68bdd84b87b028e2482c6c55b9_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:27,688 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a6c1cbcaa0714f5889ff8dcaf6db290c_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a6c1cbcaa0714f5889ff8dcaf6db290c_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:27,689 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cd59c6dec4224117995d75b82746076d_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cd59c6dec4224117995d75b82746076d_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:27,691 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ef62fe0284d344d3b4575619e35f54a1_eed6aa27f807a5ebe70b1551484242a3 to hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ef62fe0284d344d3b4575619e35f54a1_eed6aa27f807a5ebe70b1551484242a3 2024-11-14T02:46:27,691 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T02:46:27,693 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=172, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:46:27,696 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T02:46:27,699 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T02:46:27,701 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=172, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:46:27,701 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T02:46:27,701 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731552387701"}]},"ts":"9223372036854775807"} 2024-11-14T02:46:27,704 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T02:46:27,704 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => eed6aa27f807a5ebe70b1551484242a3, NAME => 'TestAcidGuarantees,,1731552352003.eed6aa27f807a5ebe70b1551484242a3.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T02:46:27,704 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T02:46:27,704 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731552387704"}]},"ts":"9223372036854775807"} 2024-11-14T02:46:27,707 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T02:46:27,708 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=172, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:46:27,709 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 82 msec 2024-11-14T02:46:27,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-14T02:46:27,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T02:46:27,745 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:46:27,762 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=238 (was 237) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=296 (was 281) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5261 (was 5272) 2024-11-14T02:46:27,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T02:46:27,762 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T02:46:27,762 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T02:46:27,762 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:27,762 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:27,762 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:46:27,762 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T02:46:27,762 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1975601807, stopped=false 2024-11-14T02:46:27,763 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=83c5a5c119d5,43239,1731552190584 2024-11-14T02:46:27,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41455-0x10138834f4e0001, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T02:46:27,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T02:46:27,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:27,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41455-0x10138834f4e0001, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:27,792 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T02:46:27,793 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T02:46:27,794 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T02:46:27,794 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:27,794 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41455-0x10138834f4e0001, quorum=127.0.0.1:60414, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T02:46:27,794 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T02:46:27,795 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '83c5a5c119d5,41455,1731552191259' ***** 2024-11-14T02:46:27,795 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T02:46:27,796 INFO [RS:0;83c5a5c119d5:41455 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T02:46:27,796 INFO [RS:0;83c5a5c119d5:41455 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T02:46:27,797 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T02:46:27,797 INFO [RS:0;83c5a5c119d5:41455 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T02:46:27,797 INFO [RS:0;83c5a5c119d5:41455 {}] regionserver.HRegionServer(959): stopping server 83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:27,797 INFO [RS:0;83c5a5c119d5:41455 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T02:46:27,797 INFO [RS:0;83c5a5c119d5:41455 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;83c5a5c119d5:41455. 2024-11-14T02:46:27,797 DEBUG [RS:0;83c5a5c119d5:41455 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T02:46:27,797 DEBUG [RS:0;83c5a5c119d5:41455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:27,798 INFO [RS:0;83c5a5c119d5:41455 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T02:46:27,798 INFO [RS:0;83c5a5c119d5:41455 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T02:46:27,798 INFO [RS:0;83c5a5c119d5:41455 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T02:46:27,798 INFO [RS:0;83c5a5c119d5:41455 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T02:46:27,799 INFO [RS:0;83c5a5c119d5:41455 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-14T02:46:27,799 DEBUG [RS:0;83c5a5c119d5:41455 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-14T02:46:27,799 DEBUG [RS:0;83c5a5c119d5:41455 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T02:46:27,799 DEBUG [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T02:46:27,799 INFO [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T02:46:27,799 DEBUG [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T02:46:27,799 DEBUG [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T02:46:27,799 DEBUG [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T02:46:27,800 INFO [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-14T02:46:27,821 DEBUG [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/hbase/meta/1588230740/.tmp/info/61b1651c2240407b9b2812c2755f677a is 95, key is TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee./info:/1731552222234/DeleteFamily/seqid=0 2024-11-14T02:46:27,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742377_1553 (size=6514) 2024-11-14T02:46:27,857 INFO [regionserver/83c5a5c119d5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T02:46:28,000 DEBUG [RS:0;83c5a5c119d5:41455 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T02:46:28,146 INFO [regionserver/83c5a5c119d5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-14T02:46:28,146 INFO [regionserver/83c5a5c119d5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-14T02:46:28,200 DEBUG [RS:0;83c5a5c119d5:41455 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T02:46:28,225 INFO [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/hbase/meta/1588230740/.tmp/info/61b1651c2240407b9b2812c2755f677a 2024-11-14T02:46:28,251 DEBUG [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/hbase/meta/1588230740/.tmp/ns/e49b7b22ba634b7487f91f7eb8708137 is 93, key is TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee./ns:/1731552222195/DeleteFamily/seqid=0 2024-11-14T02:46:28,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742378_1554 (size=5906) 2024-11-14T02:46:28,400 DEBUG [RS:0;83c5a5c119d5:41455 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T02:46:28,600 DEBUG [RS:0;83c5a5c119d5:41455 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T02:46:28,656 INFO [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/hbase/meta/1588230740/.tmp/ns/e49b7b22ba634b7487f91f7eb8708137 2024-11-14T02:46:28,680 DEBUG [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/hbase/meta/1588230740/.tmp/rep_barrier/825cf7bfed4a442e93665847eb23a9ea is 102, key is TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee./rep_barrier:/1731552222195/DeleteFamily/seqid=0 2024-11-14T02:46:28,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742379_1555 (size=6025) 2024-11-14T02:46:28,801 DEBUG [RS:0;83c5a5c119d5:41455 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T02:46:29,001 DEBUG [RS:0;83c5a5c119d5:41455 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T02:46:29,085 INFO [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/hbase/meta/1588230740/.tmp/rep_barrier/825cf7bfed4a442e93665847eb23a9ea 2024-11-14T02:46:29,130 DEBUG [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/hbase/meta/1588230740/.tmp/table/aebd81cdb63c4e878d738ea30931a556 is 96, key is TestAcidGuarantees,,1731552195383.557913855dcf97d8b784fddf23bec5ee./table:/1731552222195/DeleteFamily/seqid=0 2024-11-14T02:46:29,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742380_1556 (size=5936) 2024-11-14T02:46:29,201 DEBUG [RS:0;83c5a5c119d5:41455 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T02:46:29,402 DEBUG [RS:0;83c5a5c119d5:41455 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T02:46:29,535 INFO [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/hbase/meta/1588230740/.tmp/table/aebd81cdb63c4e878d738ea30931a556 2024-11-14T02:46:29,545 DEBUG [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/hbase/meta/1588230740/.tmp/info/61b1651c2240407b9b2812c2755f677a as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/hbase/meta/1588230740/info/61b1651c2240407b9b2812c2755f677a 2024-11-14T02:46:29,551 INFO [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/hbase/meta/1588230740/info/61b1651c2240407b9b2812c2755f677a, entries=12, sequenceid=90, filesize=6.4 K 2024-11-14T02:46:29,552 DEBUG [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/hbase/meta/1588230740/.tmp/ns/e49b7b22ba634b7487f91f7eb8708137 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/hbase/meta/1588230740/ns/e49b7b22ba634b7487f91f7eb8708137 2024-11-14T02:46:29,557 INFO [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/hbase/meta/1588230740/ns/e49b7b22ba634b7487f91f7eb8708137, entries=8, sequenceid=90, filesize=5.8 K 2024-11-14T02:46:29,558 DEBUG [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/hbase/meta/1588230740/.tmp/rep_barrier/825cf7bfed4a442e93665847eb23a9ea as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/hbase/meta/1588230740/rep_barrier/825cf7bfed4a442e93665847eb23a9ea 2024-11-14T02:46:29,563 INFO [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/hbase/meta/1588230740/rep_barrier/825cf7bfed4a442e93665847eb23a9ea, entries=6, sequenceid=90, filesize=5.9 K 2024-11-14T02:46:29,564 DEBUG [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/hbase/meta/1588230740/.tmp/table/aebd81cdb63c4e878d738ea30931a556 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/hbase/meta/1588230740/table/aebd81cdb63c4e878d738ea30931a556 2024-11-14T02:46:29,568 INFO [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/hbase/meta/1588230740/table/aebd81cdb63c4e878d738ea30931a556, entries=7, sequenceid=90, filesize=5.8 K 2024-11-14T02:46:29,569 INFO [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1770ms, sequenceid=90, compaction requested=false 2024-11-14T02:46:29,575 DEBUG [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-14T02:46:29,576 DEBUG [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T02:46:29,576 INFO [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T02:46:29,576 DEBUG [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731552387799Running coprocessor pre-close hooks at 1731552387799Disabling compacts and flushes for region at 1731552387799Disabling writes for close at 1731552387799Obtaining lock to block concurrent updates at 1731552387800 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731552387800Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731552387800Flushing stores of hbase:meta,,1.1588230740 at 1731552387800Flushing 1588230740/info: creating writer at 1731552387801 (+1 ms)Flushing 1588230740/info: appending metadata at 1731552387820 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731552387820Flushing 1588230740/ns: creating writer at 1731552388234 (+414 ms)Flushing 1588230740/ns: appending metadata at 1731552388251 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731552388251Flushing 1588230740/rep_barrier: creating writer at 1731552388666 (+415 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731552388680 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731552388680Flushing 1588230740/table: creating writer at 1731552389110 (+430 ms)Flushing 1588230740/table: appending metadata at 1731552389130 (+20 ms)Flushing 1588230740/table: closing flushed file at 1731552389130Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79cf8e0f: reopening flushed file at 1731552389544 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fab7c64: reopening flushed file at 1731552389551 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24812663: reopening flushed file at 1731552389557 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@285d0034: reopening flushed file at 1731552389563 (+6 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1770ms, sequenceid=90, compaction requested=false at 1731552389569 (+6 ms)Writing region close event to WAL at 1731552389572 (+3 ms)Running coprocessor post-close hooks at 1731552389576 (+4 ms)Closed at 1731552389576 2024-11-14T02:46:29,576 DEBUG [RS_CLOSE_META-regionserver/83c5a5c119d5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T02:46:29,602 INFO [RS:0;83c5a5c119d5:41455 {}] regionserver.HRegionServer(976): stopping server 83c5a5c119d5,41455,1731552191259; all regions closed. 2024-11-14T02:46:29,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741834_1010 (size=25344) 2024-11-14T02:46:29,613 DEBUG [RS:0;83c5a5c119d5:41455 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/oldWALs 2024-11-14T02:46:29,613 INFO [RS:0;83c5a5c119d5:41455 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 83c5a5c119d5%2C41455%2C1731552191259.meta:.meta(num 1731552194909) 2024-11-14T02:46:29,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741833_1009 (size=12917156) 2024-11-14T02:46:29,618 DEBUG [RS:0;83c5a5c119d5:41455 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/oldWALs 2024-11-14T02:46:29,618 INFO [RS:0;83c5a5c119d5:41455 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 83c5a5c119d5%2C41455%2C1731552191259:(num 1731552194303) 2024-11-14T02:46:29,618 DEBUG [RS:0;83c5a5c119d5:41455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:29,618 INFO [RS:0;83c5a5c119d5:41455 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T02:46:29,618 INFO [RS:0;83c5a5c119d5:41455 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T02:46:29,618 INFO [RS:0;83c5a5c119d5:41455 {}] hbase.ChoreService(370): Chore service for: regionserver/83c5a5c119d5:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-14T02:46:29,618 INFO [RS:0;83c5a5c119d5:41455 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T02:46:29,619 INFO [regionserver/83c5a5c119d5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T02:46:29,619 INFO [RS:0;83c5a5c119d5:41455 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41455 2024-11-14T02:46:29,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41455-0x10138834f4e0001, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/83c5a5c119d5,41455,1731552191259 2024-11-14T02:46:29,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T02:46:29,650 INFO [RS:0;83c5a5c119d5:41455 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T02:46:29,659 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [83c5a5c119d5,41455,1731552191259] 2024-11-14T02:46:29,667 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/83c5a5c119d5,41455,1731552191259 already deleted, retry=false 2024-11-14T02:46:29,667 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 83c5a5c119d5,41455,1731552191259 expired; onlineServers=0 2024-11-14T02:46:29,667 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '83c5a5c119d5,43239,1731552190584' ***** 2024-11-14T02:46:29,667 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T02:46:29,668 INFO [M:0;83c5a5c119d5:43239 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T02:46:29,668 INFO [M:0;83c5a5c119d5:43239 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T02:46:29,668 DEBUG [M:0;83c5a5c119d5:43239 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T02:46:29,668 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T02:46:29,668 DEBUG [M:0;83c5a5c119d5:43239 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T02:46:29,668 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster-HFileCleaner.small.0-1731552194015 {}] cleaner.HFileCleaner(306): Exit Thread[master/83c5a5c119d5:0:becomeActiveMaster-HFileCleaner.small.0-1731552194015,5,FailOnTimeoutGroup] 2024-11-14T02:46:29,668 DEBUG [master/83c5a5c119d5:0:becomeActiveMaster-HFileCleaner.large.0-1731552194007 {}] cleaner.HFileCleaner(306): Exit Thread[master/83c5a5c119d5:0:becomeActiveMaster-HFileCleaner.large.0-1731552194007,5,FailOnTimeoutGroup] 2024-11-14T02:46:29,669 INFO [M:0;83c5a5c119d5:43239 {}] hbase.ChoreService(370): Chore service for: master/83c5a5c119d5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T02:46:29,669 INFO [M:0;83c5a5c119d5:43239 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T02:46:29,669 DEBUG [M:0;83c5a5c119d5:43239 {}] master.HMaster(1795): Stopping service threads 2024-11-14T02:46:29,669 INFO [M:0;83c5a5c119d5:43239 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T02:46:29,669 INFO [M:0;83c5a5c119d5:43239 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T02:46:29,670 ERROR [M:0;83c5a5c119d5:43239 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:37083 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:37083,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-14T02:46:29,671 INFO [M:0;83c5a5c119d5:43239 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T02:46:29,671 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T02:46:29,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T02:46:29,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:29,675 DEBUG [M:0;83c5a5c119d5:43239 {}] zookeeper.ZKUtil(347): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T02:46:29,675 WARN [M:0;83c5a5c119d5:43239 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T02:46:29,676 INFO [M:0;83c5a5c119d5:43239 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/.lastflushedseqids 2024-11-14T02:46:29,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742381_1557 (size=72) 2024-11-14T02:46:29,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41455-0x10138834f4e0001, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T02:46:29,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41455-0x10138834f4e0001, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T02:46:29,760 INFO [RS:0;83c5a5c119d5:41455 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T02:46:29,761 INFO [RS:0;83c5a5c119d5:41455 {}] regionserver.HRegionServer(1031): Exiting; stopping=83c5a5c119d5,41455,1731552191259; zookeeper connection closed. 2024-11-14T02:46:29,761 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5bec06d5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5bec06d5 2024-11-14T02:46:29,763 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-14T02:46:30,089 INFO [M:0;83c5a5c119d5:43239 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T02:46:30,090 INFO [M:0;83c5a5c119d5:43239 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T02:46:30,090 DEBUG [M:0;83c5a5c119d5:43239 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T02:46:30,090 INFO [M:0;83c5a5c119d5:43239 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:46:30,090 DEBUG [M:0;83c5a5c119d5:43239 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:46:30,090 DEBUG [M:0;83c5a5c119d5:43239 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T02:46:30,090 DEBUG [M:0;83c5a5c119d5:43239 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:46:30,091 INFO [M:0;83c5a5c119d5:43239 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=746.72 KB heapSize=917.26 KB 2024-11-14T02:46:30,112 DEBUG [M:0;83c5a5c119d5:43239 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fb9f449c93d145e59ec65c91a8d48825 is 82, key is hbase:meta,,1/info:regioninfo/1731552194989/Put/seqid=0 2024-11-14T02:46:30,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742382_1558 (size=5672) 2024-11-14T02:46:30,517 INFO [M:0;83c5a5c119d5:43239 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2097 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fb9f449c93d145e59ec65c91a8d48825 2024-11-14T02:46:30,548 DEBUG [M:0;83c5a5c119d5:43239 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/226329df36924daeb7f18881c821eabe is 2277, key is \x00\x00\x00\x00\x00\x00\x00\x8F/proc:d/1731552355118/Put/seqid=0 2024-11-14T02:46:30,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742383_1559 (size=44543) 2024-11-14T02:46:30,916 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T02:46:30,916 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T02:46:30,918 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T02:46:30,952 INFO [M:0;83c5a5c119d5:43239 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=746.17 KB at sequenceid=2097 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/226329df36924daeb7f18881c821eabe 2024-11-14T02:46:30,955 INFO [M:0;83c5a5c119d5:43239 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 226329df36924daeb7f18881c821eabe 2024-11-14T02:46:30,973 DEBUG [M:0;83c5a5c119d5:43239 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dfef5a3c0717478e88c0c1f809ad0abf is 69, key is 83c5a5c119d5,41455,1731552191259/rs:state/1731552194061/Put/seqid=0 2024-11-14T02:46:30,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073742384_1560 (size=5156) 2024-11-14T02:46:31,378 INFO [M:0;83c5a5c119d5:43239 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2097 (bloomFilter=true), to=hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dfef5a3c0717478e88c0c1f809ad0abf 2024-11-14T02:46:31,388 DEBUG [M:0;83c5a5c119d5:43239 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fb9f449c93d145e59ec65c91a8d48825 as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fb9f449c93d145e59ec65c91a8d48825 2024-11-14T02:46:31,391 INFO [M:0;83c5a5c119d5:43239 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fb9f449c93d145e59ec65c91a8d48825, entries=8, sequenceid=2097, filesize=5.5 K 2024-11-14T02:46:31,391 DEBUG [M:0;83c5a5c119d5:43239 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/226329df36924daeb7f18881c821eabe as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/226329df36924daeb7f18881c821eabe 2024-11-14T02:46:31,394 INFO [M:0;83c5a5c119d5:43239 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 226329df36924daeb7f18881c821eabe 2024-11-14T02:46:31,394 INFO [M:0;83c5a5c119d5:43239 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/226329df36924daeb7f18881c821eabe, entries=172, sequenceid=2097, filesize=43.5 K 2024-11-14T02:46:31,395 DEBUG [M:0;83c5a5c119d5:43239 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dfef5a3c0717478e88c0c1f809ad0abf as hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/dfef5a3c0717478e88c0c1f809ad0abf 2024-11-14T02:46:31,398 INFO [M:0;83c5a5c119d5:43239 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37083/user/jenkins/test-data/d5078f5d-8856-3293-2fdc-54cb1d568d6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/dfef5a3c0717478e88c0c1f809ad0abf, entries=1, sequenceid=2097, filesize=5.0 K 2024-11-14T02:46:31,398 INFO [M:0;83c5a5c119d5:43239 {}] regionserver.HRegion(3140): Finished flush of dataSize ~746.72 KB/764645, heapSize ~916.96 KB/938968, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1308ms, sequenceid=2097, compaction requested=false 2024-11-14T02:46:31,400 INFO [M:0;83c5a5c119d5:43239 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:46:31,400 DEBUG [M:0;83c5a5c119d5:43239 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731552390090Disabling compacts and flushes for region at 1731552390090Disabling writes for close at 1731552390090Obtaining lock to block concurrent updates at 1731552390091 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731552390091Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=764645, getHeapSize=939208, getOffHeapSize=0, getCellsCount=2294 at 1731552390092 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731552390093 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731552390093Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731552390112 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731552390112Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731552390527 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731552390548 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731552390548Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731552390956 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731552390973 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731552390973Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bf29aff: reopening flushed file at 1731552391387 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@625c1bac: reopening flushed file at 1731552391391 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3985b372: reopening flushed file at 1731552391394 (+3 ms)Finished flush of dataSize ~746.72 KB/764645, heapSize ~916.96 KB/938968, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1308ms, sequenceid=2097, compaction requested=false at 1731552391398 (+4 ms)Writing region close event to WAL at 1731552391399 (+1 ms)Closed at 1731552391399 2024-11-14T02:46:31,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34599 is added to blk_1073741830_1006 (size=902157) 2024-11-14T02:46:31,403 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T02:46:31,403 INFO [M:0;83c5a5c119d5:43239 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T02:46:31,403 INFO [M:0;83c5a5c119d5:43239 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43239 2024-11-14T02:46:31,403 INFO [M:0;83c5a5c119d5:43239 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T02:46:31,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T02:46:31,526 INFO [M:0;83c5a5c119d5:43239 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T02:46:31,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43239-0x10138834f4e0000, quorum=127.0.0.1:60414, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T02:46:31,536 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T02:46:31,539 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T02:46:31,539 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T02:46:31,539 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T02:46:31,539 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/hadoop.log.dir/,STOPPED} 2024-11-14T02:46:31,542 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T02:46:31,542 WARN [BP-2060825341-172.17.0.2-1731552187113 heartbeating to localhost/127.0.0.1:37083 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T02:46:31,542 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T02:46:31,542 WARN [BP-2060825341-172.17.0.2-1731552187113 heartbeating to localhost/127.0.0.1:37083 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2060825341-172.17.0.2-1731552187113 (Datanode Uuid bc2a2627-d085-47a0-9e50-6effdfcfabbd) service to localhost/127.0.0.1:37083 2024-11-14T02:46:31,544 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/cluster_801a70f6-fc7b-44f6-e8da-0c348f54bc49/data/data1/current/BP-2060825341-172.17.0.2-1731552187113 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T02:46:31,545 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/cluster_801a70f6-fc7b-44f6-e8da-0c348f54bc49/data/data2/current/BP-2060825341-172.17.0.2-1731552187113 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T02:46:31,545 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T02:46:31,553 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T02:46:31,553 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T02:46:31,553 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T02:46:31,554 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T02:46:31,554 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d53159a0-734b-dbf5-af5c-6ecf2d141cdb/hadoop.log.dir/,STOPPED} 2024-11-14T02:46:31,572 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T02:46:31,691 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down